diff -Nru nix-2.8.0/boehmgc-coroutine-sp-fallback.diff nix-2.16.1+dfsg/boehmgc-coroutine-sp-fallback.diff --- nix-2.8.0/boehmgc-coroutine-sp-fallback.diff 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/boehmgc-coroutine-sp-fallback.diff 2023-06-06 10:55:23.000000000 +0000 @@ -1,17 +1,65 @@ +diff --git a/darwin_stop_world.c b/darwin_stop_world.c +index 0468aaec..b348d869 100644 +--- a/darwin_stop_world.c ++++ b/darwin_stop_world.c +@@ -356,6 +356,7 @@ GC_INNER void GC_push_all_stacks(void) + int nthreads = 0; + word total_size = 0; + mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ; ++ size_t stack_limit; + if (!EXPECT(GC_thr_initialized, TRUE)) + GC_thr_init(); + +@@ -411,6 +412,19 @@ GC_INNER void GC_push_all_stacks(void) + GC_push_all_stack_sections(lo, hi, p->traced_stack_sect); + } + if (altstack_lo) { ++ // When a thread goes into a coroutine, we lose its original sp until ++ // control flow returns to the thread. ++ // While in the coroutine, the sp points outside the thread stack, ++ // so we can detect this and push the entire thread stack instead, ++ // as an approximation. ++ // We assume that the coroutine has similarly added its entire stack. ++ // This could be made accurate by cooperating with the application ++ // via new functions and/or callbacks. ++ stack_limit = pthread_get_stacksize_np(p->id); ++ if (altstack_lo >= altstack_hi || altstack_lo < altstack_hi - stack_limit) { // sp outside stack ++ altstack_lo = altstack_hi - stack_limit; ++ } ++ + total_size += altstack_hi - altstack_lo; + GC_push_all_stack(altstack_lo, altstack_hi); + } +diff --git a/include/gc.h b/include/gc.h +index edab6c22..f2c61282 100644 +--- a/include/gc.h ++++ b/include/gc.h +@@ -2172,6 +2172,11 @@ GC_API void GC_CALL GC_win32_free_heap(void); + (*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_ignore_off_page) + #endif /* _AMIGA && !GC_AMIGA_MAKINGLIB */ + ++#if !__APPLE__ ++/* Patch doesn't work on apple */ ++#define NIX_BOEHM_PATCH_VERSION 1 ++#endif ++ + #ifdef __cplusplus + } /* extern "C" */ + #endif diff --git a/pthread_stop_world.c b/pthread_stop_world.c -index 4b2c429..1fb4c52 100644 +index b5d71e62..aed7b0bf 100644 --- a/pthread_stop_world.c +++ b/pthread_stop_world.c -@@ -673,6 +673,8 @@ GC_INNER void GC_push_all_stacks(void) - struct GC_traced_stack_sect_s *traced_stack_sect; - pthread_t self = pthread_self(); - word total_size = 0; +@@ -768,6 +768,8 @@ STATIC void GC_restart_handler(int sig) + /* world is stopped. Should not fail if it isn't. */ + GC_INNER void GC_push_all_stacks(void) + { + size_t stack_limit; + pthread_attr_t pattr; - - if (!EXPECT(GC_thr_initialized, TRUE)) - GC_thr_init(); -@@ -722,6 +724,31 @@ GC_INNER void GC_push_all_stacks(void) + GC_bool found_me = FALSE; + size_t nthreads = 0; + int i; +@@ -851,6 +853,31 @@ GC_INNER void GC_push_all_stacks(void) hi = p->altstack + p->altstack_size; /* FIXME: Need to scan the normal stack too, but how ? */ /* FIXME: Assume stack grows down */ diff -Nru nix-2.8.0/configure.ac nix-2.16.1+dfsg/configure.ac --- nix-2.8.0/configure.ac 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/configure.ac 2023-06-06 10:55:23.000000000 +0000 @@ -41,8 +41,6 @@ test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var -CFLAGS= -CXXFLAGS= AC_PROG_CC AC_PROG_CXX AC_PROG_CPP @@ -147,6 +145,32 @@ LDFLAGS="-latomic $LDFLAGS" fi +# Building without tests is useful for bootstrapping with a smaller footprint +# or running the tests in a separate derivation. Otherwise, we do compile and +# run them. +AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), + tests=$enableval, tests=yes) +AC_SUBST(tests) + +# Building without API docs is the default as Nix' C++ interfaces are internal and unstable. +AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), + internal_api_docs=$enableval, internal_api_docs=no) +AC_SUBST(internal_api_docs) + +# LTO is currently broken with clang for unknown reasons; ld segfaults in the llvm plugin +AC_ARG_ENABLE(lto, AS_HELP_STRING([--enable-lto],[Enable LTO (only supported with GCC) [default=no]]), + lto=$enableval, lto=no) +if test "$lto" = yes; then + if $CXX --version | grep -q GCC; then + AC_SUBST(CXXLTO, [-flto=jobserver]) + else + echo "error: LTO is only supported with GCC at the moment" >&2 + exit 1 + fi +else + AC_SUBST(CXXLTO, [""]) +fi + PKG_PROG_PKG_CONFIG AC_ARG_ENABLE(shared, AS_HELP_STRING([--enable-shared],[Build shared libraries for Nix [default=yes]]), @@ -160,10 +184,10 @@ # Look for OpenSSL, a required dependency. FIXME: this is only (maybe) # used by S3BinaryCacheStore. -PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) +PKG_CHECK_MODULES([OPENSSL], [libcrypto >= 1.1.1], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) -# Checks for libarchive +# Look for libarchive. PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"]) # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed if test "$shared" != yes; then @@ -258,10 +282,35 @@ fi +if test "$tests" = yes; then + # Look for gtest. PKG_CHECK_MODULES([GTEST], [gtest_main]) +# Look for rapidcheck. +AC_ARG_VAR([RAPIDCHECK_HEADERS], [include path of gtest headers shipped by RAPIDCHECK]) +# No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302 +AC_LANG_PUSH(C++) +AC_SUBST(RAPIDCHECK_HEADERS) +[CXXFLAGS="-I $RAPIDCHECK_HEADERS $CXXFLAGS"] +[LIBS="-lrapidcheck -lgtest $LIBS"] +AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include ]) +dnl AC_CHECK_LIB doesn't work for C++ libs with mangled symbols +AC_LINK_IFELSE([ + AC_LANG_PROGRAM([[ + #include + #include + ]], [[ + return RUN_ALL_TESTS(); + ]]) + ], + [], + [AC_MSG_ERROR([librapidcheck is not found.])]) +AC_LANG_POP(C++) + +fi + # Look for nlohmann/json. PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9]) @@ -282,18 +331,28 @@ AC_CHECK_FUNCS([strsignal posix_fallocate sysconf]) -# This is needed if bzip2 is a static library, and the Nix libraries -# are dynamic. -case "${host_os}" in - darwin*) - LDFLAGS="-all_load $LDFLAGS" - ;; -esac - - AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]), sandbox_shell=$withval) AC_SUBST(sandbox_shell) +if test ${cross_compiling:-no} = no && ! test -z ${sandbox_shell+x}; then + AC_MSG_CHECKING([whether sandbox-shell has the standalone feature]) + # busybox shell sometimes allows executing other busybox applets, + # even if they are not in the path, breaking our sandbox + if PATH= $sandbox_shell -c "busybox" 2>&1 | grep -qv "not found"; then + AC_MSG_RESULT(enabled) + AC_MSG_ERROR([Please disable busybox FEATURE_SH_STANDALONE]) + else + AC_MSG_RESULT(disabled) + fi +fi + +AC_ARG_ENABLE(embedded-sandbox-shell, AS_HELP_STRING([--enable-embedded-sandbox-shell],[include the sandbox shell in the Nix binary [default=no]]), + embedded_sandbox_shell=$enableval, embedded_sandbox_shell=no) +AC_SUBST(embedded_sandbox_shell) +if test "$embedded_sandbox_shell" = yes; then + AC_DEFINE(HAVE_EMBEDDED_SANDBOX_SHELL, 1, [Include the sandbox shell in the Nix binary.]) +fi + # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix diff -Nru nix-2.8.0/CONTRIBUTING.md nix-2.16.1+dfsg/CONTRIBUTING.md --- nix-2.8.0/CONTRIBUTING.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/CONTRIBUTING.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,62 @@ +# Contributing to Nix + +Welcome and thank you for your interest in contributing to Nix! +We appreciate your support. + +Reading and following these guidelines will help us make the contribution process easy and effective for everyone involved. + + +## Report a bug + +1. Check on the [GitHub issue tracker](https://github.com/NixOS/nix/issues) if your bug was already reported. + +2. If you were not able to find the bug or feature [open a new issue](https://github.com/NixOS/nix/issues/new/choose) + +3. The issue templates will guide you in specifying your issue. + The more complete the information you provide, the more likely it can be found by others and the more useful it is in the future. + Make sure reported bugs can be reproduced easily. + +4. Once submitted, do not expect issues to be picked up or solved right away. + The only way to ensure this, is to [work on the issue yourself](#making-changes-to-nix). + +## Report a security vulnerability + +Check out the [security policy](https://github.com/NixOS/nix/security/policy). + +## Making changes to Nix + +1. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. + There are many open pull requests that might already do what you intent to work on. + You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. + +2. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue. + Pull requests addressing issues labeled ["idea approved"](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review. + +3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. + + For contributions to the command line interface, please check the [CLI guidelines](https://nixos.org/manual/nix/unstable/contributing/cli-guideline.html). + +4. Make your changes! + +5. [Create a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) for your changes. + * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. + * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). + * Link related issues in your pull request to inform interested parties and future contributors about your change. + If your pull request closes one or multiple issues, note that in the description using `Closes: #`, as it will then happen automatically when your change is merged. + +6. Do not expect your pull request to be reviewed immediately. + Nix maintainers follow a [structured process for reviews and design decisions](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol), which may or may not prioritise your work. + +7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams). + +## Making changes to the Nix manual + +The Nix reference manual is hosted on https://nixos.org/manual/nix. +The underlying source files are located in [`doc/manual/src`](./doc/manual/src). +For small changes you can [use GitHub to edit these files](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files) +For larger changes see the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html). + +## Getting help + +Whenever you're stuck or do not know how to proceed, you can always ask for help. +The appropriate channels to do so can be found on the [NixOS Community](https://nixos.org/community/) page. diff -Nru nix-2.8.0/debian/changelog nix-2.16.1+dfsg/debian/changelog --- nix-2.8.0/debian/changelog 2022-11-08 03:05:26.000000000 +0000 +++ nix-2.16.1+dfsg/debian/changelog 2023-10-04 02:33:09.000000000 +0000 @@ -1,15 +1,47 @@ -nix (2.8.0-1.1build1) lunar; urgency=medium +nix (2.16.1+dfsg-3ubuntu1) mantic; urgency=medium - * No-change rebuild against libcpuid16 + * d/control: add libssh-dev dependency (LP: #2037314). + * d/t/control: disable tests requiring internet access. - -- Steve Langasek Tue, 08 Nov 2022 03:05:26 +0000 + -- Vladimir Petko Wed, 04 Oct 2023 15:33:09 +1300 -nix (2.8.0-1.1) unstable; urgency=medium +nix (2.16.1+dfsg-3) unstable; urgency=medium - * Non-maintainer upload. - * No source change upload to rebuild with debhelper 13.10. + [ Jordan Justen ] + * Release to unstable - -- Michael Biebl Sat, 15 Oct 2022 12:28:28 +0200 + [ Adrian Bunk ] + * Use -g1 to work around mipsel address space limitation + + -- Jordan Justen Mon, 07 Aug 2023 02:29:40 -0700 + +nix (2.16.1+dfsg-2) unstable; urgency=medium + + * d/patches: Fix build error for !amd64 (Closes: #1041876) + + -- Jordan Justen Sat, 29 Jul 2023 01:41:24 -0700 + +nix (2.16.1+dfsg-1) unstable; urgency=medium + + * New upstream release (Closes: #1037790) + * d/control: Assign Jordan Justen as Maintainer (Closes: #1019013) + * d/watch: Scan github for new tags and tarballs + * d/patches: Rebase patches for 2.16.1 + * d/control: Add librapidcheck-dev build-dep + * d/rules: Drop quotes around EDITLINE_CFLAGS/EDITLINE_LIBS definition + * d/rules: configure with --disable-shared to link to librapidcheck + * d/control: Add new build dependencies for 2.16.1 + * d/*install: Update dh_install configuration for 2.16.1 + * d/copyright: Strip files from github tag based tarball + * d/source/options: Ignore files stripped from tarball + * d/patches: Don't use bundled toml11 (Closes: #1005879, #1005880) + * d/watch, d/changelog: Add dfsg suffix back + * d/control: Drop lsb-base dependency (lintian depends-on-obsolete-package) + * d/control: Update Standards-Version to 4.6.2 + * d/nix-bin.lintian-overrides: Drop `nam` spelling-error-in-binary no longer flagged + * d/nix-bin.lintian-overrides: Update spare-manual-page override format + + -- Jordan Justen Mon, 24 Jul 2023 01:12:23 -0700 nix (2.8.0-1) unstable; urgency=medium diff -Nru nix-2.8.0/debian/control nix-2.16.1+dfsg/debian/control --- nix-2.8.0/debian/control 2022-11-08 03:05:26.000000000 +0000 +++ nix-2.16.1+dfsg/debian/control 2023-10-04 02:33:09.000000000 +0000 @@ -2,13 +2,14 @@ Section: devel Priority: optional Maintainer: Ubuntu Developers -XSBC-Original-Maintainer: Thomas Koch +XSBC-Original-Maintainer: Jordan Justen Build-Depends: debhelper-compat (= 13), autoconf-archive, bison, curl, flex, jq, + libacl1-dev, libarchive-dev, libboost-dev, libboost-context-dev, @@ -22,20 +23,36 @@ libdbd-sqlite3-perl, libgc-dev, libgmock-dev, + libgnutls28-dev, libgtest-dev, + libidn2-dev, + libkrb5-dev, + libldap2-dev, liblowdown-dev, + liblz4-dev, liblzma-dev, + libnghttp2-dev, + libpsl-dev, + librapidcheck-dev, libreadline-dev, + librtmp-dev, libseccomp-dev, + libssh-dev, + libssh2-1-dev, libsodium-dev, libsqlite3-dev, libssl-dev, + libtoml11-dev, libwww-curl-perl, + libxml2-dev, libxml2-utils, + libzstd-dev, lowdown, + nettle-dev, nlohmann-json3-dev, pkg-config, -Standards-Version: 4.6.0 + zlib1g-dev, +Standards-Version: 4.6.2 Homepage: https://nixos.org/nix Vcs-Browser: https://salsa.debian.org/debian/nix Vcs-Git: https://salsa.debian.org/debian/nix.git @@ -47,7 +64,6 @@ curl, libdbd-sqlite3-perl, libwww-curl-perl, - lsb-base (>= 3.0-6), perl, xz-utils, ${shlibs:Depends}, ${misc:Depends}, ${perl:Depends} diff -Nru nix-2.8.0/debian/copyright nix-2.16.1+dfsg/debian/copyright --- nix-2.8.0/debian/copyright 2022-05-02 18:55:23.000000000 +0000 +++ nix-2.16.1+dfsg/debian/copyright 2023-10-04 02:33:09.000000000 +0000 @@ -4,8 +4,19 @@ Comment: Reasons for file exclusions - eval-okay-fromjson.nix contains an example from an RFC file see https://github.com/NixOS/nix/issues/3391 + - scripts/install-multi-user.sh is sourced from many places. Also the other + install scripts are not needed in a Debian package. - exclude bundled dependency json.hpp (package nlohmann-json3-dev) - exclude bundled dependency highlight.js + - exclude bundled dependency toml11 +Files-Excluded: + tests/lang/eval-okay-fromjson.* + scripts/create-darwin-volume.sh + scripts/install-darwin-multi-user.sh + scripts/install-multi-user.sh + scripts/install-nix-from-closure.sh + doc/manual/theme + src/toml11 Files: * Copyright: 2006-2020 Eelco Dostra @@ -79,11 +90,6 @@ and this notice are preserved. This file is offered as-is, without any warranty. -Files: src/toml11/* -Copyright: Copyright (c) 2017 Toru Niina -License: MIT -Comment: See https://bugs.debian.org/1004498 for a RFP - Files: debian/* Copyright: 2017-2018 Kai Harries 2018-2022 Thomas Koch diff -Nru nix-2.8.0/debian/nix-bin.install nix-2.16.1+dfsg/debian/nix-bin.install --- nix-2.8.0/debian/nix-bin.install 2022-05-02 18:55:23.000000000 +0000 +++ nix-2.16.1+dfsg/debian/nix-bin.install 2023-10-04 02:33:09.000000000 +0000 @@ -3,4 +3,5 @@ usr/libexec/nix/build-remote usr/share/bash-completion/completions/nix usr/share/fish/vendor_completions.d/nix.fish usr/share/fish/completions -usr/share/zsh/site-functions/_nix usr/share/zsh/vendor-completions \ No newline at end of file +usr/share/zsh/site-functions/_nix usr/share/zsh/vendor-completions +usr/share/zsh/site-functions/run-help-nix usr/share/zsh/vendor-completions diff -Nru nix-2.8.0/debian/nix-bin.lintian-overrides nix-2.16.1+dfsg/debian/nix-bin.lintian-overrides --- nix-2.8.0/debian/nix-bin.lintian-overrides 2022-05-02 18:56:22.000000000 +0000 +++ nix-2.16.1+dfsg/debian/nix-bin.lintian-overrides 2023-10-04 02:33:09.000000000 +0000 @@ -1,5 +1,2 @@ # These manual pages document subcommands of the thix command -nix-bin binary: spare-manual-page usr/share/man/man1/nix3-* - -# I tried my best but could not find this string -nix-bin binary: spelling-error-in-binary usr/lib/nix/libnixcmd.so nam name \ No newline at end of file +nix-bin binary: spare-manual-page [usr/share/man/man1/nix3-*] diff -Nru nix-2.8.0/debian/not-installed nix-2.16.1+dfsg/debian/not-installed --- nix-2.8.0/debian/not-installed 2022-05-02 18:56:22.000000000 +0000 +++ nix-2.16.1+dfsg/debian/not-installed 2023-10-04 02:33:09.000000000 +0000 @@ -1,6 +1,8 @@ etc/init/nix-daemon.conf etc/profile.d/nix-daemon.sh etc/profile.d/nix.sh +etc/profile.d/nix-daemon.fish +etc/profile.d/nix.fish lib/tmpfiles.d/nix-daemon.conf usr/include/* -usr/lib/pkgconfig/* \ No newline at end of file +usr/lib/pkgconfig/* diff -Nru nix-2.8.0/debian/patches/0001-fix-service-file-path.patch nix-2.16.1+dfsg/debian/patches/0001-fix-service-file-path.patch --- nix-2.8.0/debian/patches/0001-fix-service-file-path.patch 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/debian/patches/0001-fix-service-file-path.patch 2023-10-04 02:33:09.000000000 +0000 @@ -0,0 +1,23 @@ +From: Thomas Koch +Date: Sun, 16 Jul 2023 00:14:20 -0700 +Subject: fix-service-file-path + +Install systemd units into /lib instead of /usr/lib +--- + misc/systemd/local.mk | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/misc/systemd/local.mk b/misc/systemd/local.mk +index 76121a0..a931269 100644 +--- a/misc/systemd/local.mk ++++ b/misc/systemd/local.mk +@@ -1,7 +1,7 @@ + ifdef HOST_LINUX + +- $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644))) +- $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/tmpfiles.d, 0644))) ++ $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), /lib/systemd/system, 0644))) ++ $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), /lib/tmpfiles.d, 0644))) + + clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service $(d)/nix-daemon.conf + diff -Nru nix-2.8.0/debian/patches/0002-Append-CPPFLAGS-to-GLOBAL_-FLAGS-copy-LDFLAGS-into-G.patch nix-2.16.1+dfsg/debian/patches/0002-Append-CPPFLAGS-to-GLOBAL_-FLAGS-copy-LDFLAGS-into-G.patch --- nix-2.8.0/debian/patches/0002-Append-CPPFLAGS-to-GLOBAL_-FLAGS-copy-LDFLAGS-into-G.patch 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/debian/patches/0002-Append-CPPFLAGS-to-GLOBAL_-FLAGS-copy-LDFLAGS-into-G.patch 2023-10-04 02:33:09.000000000 +0000 @@ -0,0 +1,25 @@ +From: Kai Harries +Date: Sun, 16 Jul 2023 00:14:20 -0700 +Subject: Append CPPFLAGS to GLOBAL_*FLAGS, copy LDFLAGS into GLOBAL_LDFLAGS + +I (thk@) am not sure what this patch is needed for. I guess so that Debian +specific build options are respected? +--- + Makefile | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/Makefile b/Makefile +index d6b4947..6e17a64 100644 +--- a/Makefile ++++ b/Makefile +@@ -42,6 +42,10 @@ else + GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE + endif + ++GLOBAL_CFLAGS += $(CPPFLAGS) ++GLOBAL_CXXFLAGS += $(CPPFLAGS) ++GLOBAL_LDFLAGS += $(LDFLAGS) ++ + include mk/lib.mk + + GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src diff -Nru nix-2.8.0/debian/patches/0003-disable-use-of-mdbook.patch nix-2.16.1+dfsg/debian/patches/0003-disable-use-of-mdbook.patch --- nix-2.8.0/debian/patches/0003-disable-use-of-mdbook.patch 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/debian/patches/0003-disable-use-of-mdbook.patch 2023-10-04 02:33:09.000000000 +0000 @@ -0,0 +1,56 @@ +From: Thomas Koch +Date: Sun, 16 Jul 2023 00:14:20 -0700 +Subject: disable use of mdbook + +Forwarded: not-needed +Bug-Debian: https://bugs.debian.org/1004481 +--- + doc/manual/local.mk | 36 +++++++++++++++++++----------------- + 1 file changed, 19 insertions(+), 17 deletions(-) + +diff --git a/doc/manual/local.mk b/doc/manual/local.mk +index b4b7283..9f2b562 100644 +--- a/doc/manual/local.mk ++++ b/doc/manual/local.mk +@@ -168,22 +168,24 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli + @touch $@ + + $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md +- $(trace-gen) \ +- tmp="$$(mktemp -d)"; \ +- cp -r doc/manual "$$tmp"; \ +- find "$$tmp" -name '*.md' | while read -r file; do \ +- $(call process-includes,$$file,$$file); \ +- done; \ +- find "$$tmp" -name '*.md' | while read -r file; do \ +- docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/src)"; \ +- sed -i "s,@docroot@,$$docroot,g" "$$file"; \ +- done; \ +- set -euo pipefail; \ +- RUST_LOG=warn mdbook build "$$tmp/manual" -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \ +- | { grep -Fv "because fragment resolution isn't implemented" || :; }; \ +- rm -rf "$$tmp/manual" +- @rm -rf $(DESTDIR)$(docdir)/manual +- @mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual +- @rm -rf $(DESTDIR)$(docdir)/manual.tmp ++ # $(trace-gen) \ ++ # tmp="$$(mktemp -d)"; \ ++ # cp -r doc/manual "$$tmp"; \ ++ # find "$$tmp" -name '*.md' | while read -r file; do \ ++ # $(call process-includes,$$file,$$file); \ ++ # done; \ ++ # find "$$tmp" -name '*.md' | while read -r file; do \ ++ # docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/src)"; \ ++ # sed -i "s,@docroot@,$$docroot,g" "$$file"; \ ++ # done; \ ++ # set -euo pipefail; \ ++ # RUST_LOG=warn mdbook build "$$tmp/manual" -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \ ++ # | { grep -Fv "because fragment resolution isn't implemented" || :; }; \ ++ # rm -rf "$$tmp/manual" ++ # @rm -rf $(DESTDIR)$(docdir)/manual ++ # @mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual ++ # @rm -rf $(DESTDIR)$(docdir)/manual.tmp ++ @echo "skipping mdbook since mdbook is not yet packaged for Debian, see:" ++ @echo "https://github.com/NixOS/nixpkgs/blob/master/pkgs/tools/text/mdbook/default.nix" + + endif diff -Nru nix-2.8.0/debian/patches/0004-Fix-build-phase-depending-on-installed-binary.patch nix-2.16.1+dfsg/debian/patches/0004-Fix-build-phase-depending-on-installed-binary.patch --- nix-2.8.0/debian/patches/0004-Fix-build-phase-depending-on-installed-binary.patch 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/debian/patches/0004-Fix-build-phase-depending-on-installed-binary.patch 2023-10-04 02:33:09.000000000 +0000 @@ -0,0 +1,83 @@ +From: Thomas Koch +Date: Sun, 16 Jul 2023 00:14:20 -0700 +Subject: Fix build phase depending on installed binary + +Bug: https://github.com/NixOS/nix/issues/5781 +Forwarded: https://github.com/NixOS/nix/pull/6015 +--- + doc/manual/local.mk | 28 ++++++++++++++-------------- + 1 file changed, 14 insertions(+), 14 deletions(-) + +diff --git a/doc/manual/local.mk b/doc/manual/local.mk +index 9f2b562..ac36cb7 100644 +--- a/doc/manual/local.mk ++++ b/doc/manual/local.mk +@@ -32,7 +32,7 @@ dummy-env = env -i \ + NIX_STATE_DIR=/dummy \ + NIX_CONFIG='cores = 0' + +-nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw ++nix-eval = $(dummy-env) $(nix_DIR)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw + + # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution + define process-includes +@@ -96,46 +96,46 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/sr + @cp $< $@ + @$(call process-includes,$@,$@) + +-$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix ++$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(nix_DIR)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' + @mv $@.tmp $@ + +-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix ++$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(nix_DIR)/nix + @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp + $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + @mv $@.tmp $@ + +-$(d)/nix.json: $(bindir)/nix +- $(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp ++$(d)/nix.json: $(nix_DIR)/nix ++ $(trace-gen) $(dummy-env) $(nix_DIR)/nix __dump-cli > $@.tmp + @mv $@.tmp $@ + +-$(d)/conf-file.json: $(bindir)/nix +- $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp ++$(d)/conf-file.json: $(nix_DIR)/nix ++ $(trace-gen) $(dummy-env) $(nix_DIR)/nix show-config --json --experimental-features nix-command > $@.tmp + @mv $@.tmp $@ + +-$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix ++$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(nix_DIR)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + +-$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix ++$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(nix_DIR)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + +-$(d)/xp-features.json: $(bindir)/nix +- $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp ++$(d)/xp-features.json: $(nix_DIR)/nix ++ $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(nix_DIR)/nix __dump-xp-features > $@.tmp + @mv $@.tmp $@ + +-$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix ++$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(nix_DIR)/nix + @cat doc/manual/src/language/builtins-prefix.md > $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp + @mv $@.tmp $@ + +-$(d)/builtins.json: $(bindir)/nix +- $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp ++$(d)/builtins.json: $(nix_DIR)/nix ++ $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(nix_DIR)/nix __dump-builtins > $@.tmp + @mv $@.tmp $@ + + # Generate the HTML manual. diff -Nru nix-2.8.0/debian/patches/0005-lowdown-debian-package-does-not-declare-dependency-o.patch nix-2.16.1+dfsg/debian/patches/0005-lowdown-debian-package-does-not-declare-dependency-o.patch --- nix-2.8.0/debian/patches/0005-lowdown-debian-package-does-not-declare-dependency-o.patch 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/debian/patches/0005-lowdown-debian-package-does-not-declare-dependency-o.patch 2023-10-04 02:33:09.000000000 +0000 @@ -0,0 +1,23 @@ +From: Thomas Koch +Date: Sun, 16 Jul 2023 00:14:20 -0700 +Subject: lowdown debian package does not declare dependency on libbsd + +Forwarded: not-needed +Bug: https://bugs.debian.org/1004541 +--- + src/libcmd/local.mk | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk +index 541a7d2..9029af5 100644 +--- a/src/libcmd/local.mk ++++ b/src/libcmd/local.mk +@@ -8,7 +8,7 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc) + + libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers + +-libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread ++libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -lbsd -pthread + + libcmd_LIBS = libstore libutil libexpr libmain libfetchers + diff -Nru nix-2.8.0/debian/patches/0006-libexpr-Don-t-use-bundled-toml11.patch nix-2.16.1+dfsg/debian/patches/0006-libexpr-Don-t-use-bundled-toml11.patch --- nix-2.8.0/debian/patches/0006-libexpr-Don-t-use-bundled-toml11.patch 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/debian/patches/0006-libexpr-Don-t-use-bundled-toml11.patch 2023-10-04 02:33:09.000000000 +0000 @@ -0,0 +1,27 @@ +From: Jordan Justen +Date: Wed, 19 Jul 2023 10:13:01 -0700 +Subject: libexpr: Don't use bundled toml11 + +Instead use toml11 from distro. + +Forwarded: not-needed +Bug: https://bugs.debian.org/1005879 +Bug: https://bugs.debian.org/1005880 +Signed-off-by: Jordan Justen +--- + src/libexpr/primops/fromTOML.cc | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc +index 8a52317..0ef92d4 100644 +--- a/src/libexpr/primops/fromTOML.cc ++++ b/src/libexpr/primops/fromTOML.cc +@@ -1,7 +1,7 @@ + #include "primops.hh" + #include "eval-inline.hh" + +-#include "../../toml11/toml.hpp" ++#include "toml.hpp" + + namespace nix { + diff -Nru nix-2.8.0/debian/patches/0007-configure.ac-Skip-I-for-RAPIDCHECK_HEADERS-when-not-.patch nix-2.16.1+dfsg/debian/patches/0007-configure.ac-Skip-I-for-RAPIDCHECK_HEADERS-when-not-.patch --- nix-2.8.0/debian/patches/0007-configure.ac-Skip-I-for-RAPIDCHECK_HEADERS-when-not-.patch 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/debian/patches/0007-configure.ac-Skip-I-for-RAPIDCHECK_HEADERS-when-not-.patch 2023-10-04 02:33:09.000000000 +0000 @@ -0,0 +1,40 @@ +From: Jordan Justen +Date: Sat, 29 Jul 2023 01:14:32 -0700 +Subject: configure.ac: Skip -I for RAPIDCHECK_HEADERS when not set + +I was seeing build output which looked like: + +g++ -o src/libcmd/repl.o -c src/libcmd/repl.cc -I src -I -DREADLINE +src/libcmd/repl.cc:18:10: fatal error: editline.h: No such file or directory + 18 | #include + | ^~~~~~~~~~~~ +compilation terminated. + +Notice the second -I is not followed by a path. I think this caused +the -DREADLINE to be interpreted as the path, and therefore the +READLINE macro wasn't defined. This lead to the incorrect +pre-processing of src/libcmd/repl.cc. + +I did not test the case where the RAPIDCHECK_HEADERS variable was set. + +Signed-off-by: Jordan Justen +Forwarded: https://github.com/NixOS/nix/pull/8757 +--- + configure.ac | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/configure.ac b/configure.ac +index bb3f92e..cbaaa43 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -293,7 +293,9 @@ AC_ARG_VAR([RAPIDCHECK_HEADERS], [include path of gtest headers shipped by RAPID + # No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302 + AC_LANG_PUSH(C++) + AC_SUBST(RAPIDCHECK_HEADERS) +-[CXXFLAGS="-I $RAPIDCHECK_HEADERS $CXXFLAGS"] ++if test "x$RAPIDCHECK_HEADERS" != "x"; then ++ [CXXFLAGS="-I $RAPIDCHECK_HEADERS $CXXFLAGS"] ++fi + [LIBS="-lrapidcheck -lgtest $LIBS"] + AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include ]) + dnl AC_CHECK_LIB doesn't work for C++ libs with mangled symbols diff -Nru nix-2.8.0/debian/patches/disable-mdbook.patch nix-2.16.1+dfsg/debian/patches/disable-mdbook.patch --- nix-2.8.0/debian/patches/disable-mdbook.patch 2022-05-02 18:55:23.000000000 +0000 +++ nix-2.16.1+dfsg/debian/patches/disable-mdbook.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,17 +0,0 @@ -Subject: disable use of mdbook -Author: Thomas Koch -Forwarded: not-needed -Bug-Debian: https://bugs.debian.org/1004481 - ---- a/doc/manual/local.mk -+++ b/doc/manual/local.mk -@@ -98,6 +98,8 @@ doc/manual/generated/man1/nix3-manpages: - @touch $@ - - $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(call rwildcard, $(d)/src, *.md) -- $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual -+# $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual -+ @echo "skipping mdbook since mdbook is not yet packaged for Debian, see:" -+ @echo "https://github.com/NixOS/nixpkgs/blob/master/pkgs/tools/text/mdbook/default.nix" - - endif diff -Nru nix-2.8.0/debian/patches/fix-Makefile.patch nix-2.16.1+dfsg/debian/patches/fix-Makefile.patch --- nix-2.8.0/debian/patches/fix-Makefile.patch 2022-05-02 18:55:23.000000000 +0000 +++ nix-2.16.1+dfsg/debian/patches/fix-Makefile.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,19 +0,0 @@ -Subject: Append CPPFLAGS to GLOBAL_*FLAGS, copy LDFLAGS into GLOBAL_LDFLAGS -Author: Kai Harries - -I (thk@) am not sure what this patch is needed for. I guess so that Debian -specific build options are respected? - ---- a/Makefile -+++ b/Makefile -@@ -32,6 +32,10 @@ else - GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE - endif - -+GLOBAL_CFLAGS += $(CPPFLAGS) -+GLOBAL_CXXFLAGS += $(CPPFLAGS) -+GLOBAL_LDFLAGS += $(LDFLAGS) -+ - include mk/lib.mk - - GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 -I src diff -Nru nix-2.8.0/debian/patches/fix_nix_DIR_in_doc_local_mk.patch nix-2.16.1+dfsg/debian/patches/fix_nix_DIR_in_doc_local_mk.patch --- nix-2.8.0/debian/patches/fix_nix_DIR_in_doc_local_mk.patch 2022-05-02 18:55:23.000000000 +0000 +++ nix-2.16.1+dfsg/debian/patches/fix_nix_DIR_in_doc_local_mk.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,57 +0,0 @@ -Subject: Fix build phase depending on installed binary -Author: Thomas Koch -Bug: https://github.com/NixOS/nix/issues/5781 -Forwarded: https://github.com/NixOS/nix/pull/6015 - ---- a/doc/manual/local.mk -+++ b/doc/manual/local.mk -@@ -20,7 +20,7 @@ dummy-env = env -i \ - NIX_STATE_DIR=/dummy \ - NIX_CONFIG='cores = 0' - --nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw -+nix-eval = $(dummy-env) $(nix_DIR)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw - - $(d)/%.1: $(d)/src/command-ref/%.md - @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp -@@ -44,31 +44,31 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md - $(trace-gen) cat doc/manual/src/SUMMARY.md.in | while IFS= read line; do if [[ $$line = @manpages@ ]]; then cat doc/manual/src/command-ref/new-cli/SUMMARY.md; else echo "$$line"; fi; done > $@.tmp - @mv $@.tmp $@ - --$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix -+$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(nix_DIR)/nix - @rm -rf $@ - $(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { command = builtins.readFile $<; renderLinks = true; }' - --$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix -+$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(nix_DIR)/nix - @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp - @mv $@.tmp $@ - --$(d)/nix.json: $(bindir)/nix -- $(trace-gen) $(dummy-env) $(bindir)/nix __dump-args > $@.tmp -+$(d)/nix.json: $(nix_DIR)/nix -+ $(trace-gen) $(dummy-env) $(nix_DIR)/nix __dump-args > $@.tmp - @mv $@.tmp $@ - --$(d)/conf-file.json: $(bindir)/nix -- $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp -+$(d)/conf-file.json: $(nix_DIR)/nix -+ $(trace-gen) $(dummy-env) $(nix_DIR)/nix show-config --json --experimental-features nix-command > $@.tmp - @mv $@.tmp $@ - --$(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix -+$(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(nix_DIR)/nix - @cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp - @cat doc/manual/src/expressions/builtins-suffix.md >> $@.tmp - @mv $@.tmp $@ - --$(d)/builtins.json: $(bindir)/nix -- $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp -+$(d)/builtins.json: $(nix_DIR)/nix -+ $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(nix_DIR)/nix __dump-builtins > $@.tmp - @mv $@.tmp $@ - - # Generate the HTML manual. diff -Nru nix-2.8.0/debian/patches/fix-service-file-path.patch nix-2.16.1+dfsg/debian/patches/fix-service-file-path.patch --- nix-2.8.0/debian/patches/fix-service-file-path.patch 2022-05-02 18:56:22.000000000 +0000 +++ nix-2.16.1+dfsg/debian/patches/fix-service-file-path.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,13 +0,0 @@ -Install systemd units into /lib instead of /usr/lib ---- a/misc/systemd/local.mk -+++ b/misc/systemd/local.mk -@@ -1,7 +1,7 @@ - ifdef HOST_LINUX - -- $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644))) -- $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/tmpfiles.d, 0644))) -+ $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), /lib/systemd/system, 0644))) -+ $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), /lib/tmpfiles.d, 0644))) - - clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service $(d)/nix-daemon.conf - diff -Nru nix-2.8.0/debian/patches/lowdown_does_not_declare_bsd_dep.patch nix-2.16.1+dfsg/debian/patches/lowdown_does_not_declare_bsd_dep.patch --- nix-2.8.0/debian/patches/lowdown_does_not_declare_bsd_dep.patch 2022-05-02 18:55:23.000000000 +0000 +++ nix-2.16.1+dfsg/debian/patches/lowdown_does_not_declare_bsd_dep.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,16 +0,0 @@ -Subject: lowdown debian package does not declare dependency on libbsd -Author: Thomas Koch -Forwarded: not-needed -Bug: https://bugs.debian.org/1004541 - ---- a/src/libcmd/local.mk -+++ b/src/libcmd/local.mk -@@ -8,7 +8,7 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc) - - libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers - --libcmd_LDFLAGS += $(LOWDOWN_LIBS) -pthread -+libcmd_LDFLAGS += $(LOWDOWN_LIBS) -lbsd -pthread - - libcmd_LIBS = libstore libutil libexpr libmain libfetchers - diff -Nru nix-2.8.0/debian/patches/series nix-2.16.1+dfsg/debian/patches/series --- nix-2.8.0/debian/patches/series 2022-05-02 18:55:23.000000000 +0000 +++ nix-2.16.1+dfsg/debian/patches/series 2023-10-04 02:33:09.000000000 +0000 @@ -1,6 +1,7 @@ -fix-service-file-path.patch -fix-Makefile.patch -disable-mdbook.patch -fix_nix_DIR_in_doc_local_mk.patch - -lowdown_does_not_declare_bsd_dep.patch +0001-fix-service-file-path.patch +0002-Append-CPPFLAGS-to-GLOBAL_-FLAGS-copy-LDFLAGS-into-G.patch +0003-disable-use-of-mdbook.patch +0004-Fix-build-phase-depending-on-installed-binary.patch +0005-lowdown-debian-package-does-not-declare-dependency-o.patch +0006-libexpr-Don-t-use-bundled-toml11.patch +0007-configure.ac-Skip-I-for-RAPIDCHECK_HEADERS-when-not-.patch diff -Nru nix-2.8.0/debian/rules nix-2.16.1+dfsg/debian/rules --- nix-2.8.0/debian/rules 2022-05-02 18:55:23.000000000 +0000 +++ nix-2.16.1+dfsg/debian/rules 2023-10-04 02:33:09.000000000 +0000 @@ -20,8 +20,12 @@ export DEB_LDFLAGS_MAINT_APPEND = -Wl,--no-as-needed -latomic -Wl,--as-needed endif -export EDITLINE_CFLAGS = "-DREADLINE" -export EDITLINE_LIBS = "/usr/lib/$(DEB_HOST_MULTIARCH)/libhistory.so /usr/lib/$(DEB_HOST_MULTIARCH)/libreadline.so" +ifneq (,$(filter $(DEB_HOST_ARCH), mipsel)) + export DEB_CXXFLAGS_MAINT_APPEND += -g1 +endif + +export EDITLINE_CFLAGS = -DREADLINE +export EDITLINE_LIBS = /usr/lib/$(DEB_HOST_MULTIARCH)/libhistory.so /usr/lib/$(DEB_HOST_MULTIARCH)/libreadline.so export V=1 export GLOBAL_CXXFLAGS += -fstack-protector-strong -Wformat -Werror=format-security -ffile-prefix-map=$(CURDIR)=. @@ -34,6 +38,7 @@ override_dh_auto_configure: dh_auto_configure -- \ --enable-gc \ + --disable-shared \ --with-coreutils-bin=/usr/bin:/bin \ --localstatedir=/nix/var \ --libdir=/usr/lib/nix diff -Nru nix-2.8.0/debian/source/options nix-2.16.1+dfsg/debian/source/options --- nix-2.8.0/debian/source/options 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/debian/source/options 2023-10-04 02:33:09.000000000 +0000 @@ -0,0 +1,17 @@ +tar-ignore = "doc/manual/theme/highlight.js" +tar-ignore = "scripts/create-darwin-volume.sh" +tar-ignore = "scripts/install-darwin-multi-user.sh" +tar-ignore = "scripts/install-multi-user.sh" +tar-ignore = "scripts/install-nix-from-closure.sh" +tar-ignore = "src/toml11" +tar-ignore = "tests/lang/eval-okay-fromjson.exp" +tar-ignore = "tests/lang/eval-okay-fromjson.nix" + +extend-diff-ignore = "^doc/manual/theme/highlight.js$" +extend-diff-ignore = "^scripts/create-darwin-volume.sh$" +extend-diff-ignore = "^scripts/install-darwin-multi-user.sh$" +extend-diff-ignore = "^scripts/install-multi-user.sh$" +extend-diff-ignore = "^scripts/install-nix-from-closure.sh$" +extend-diff-ignore = "^src/toml11/.+$" +extend-diff-ignore = "^tests/lang/eval-okay-fromjson.exp$" +extend-diff-ignore = "^tests/lang/eval-okay-fromjson.nix$" diff -Nru nix-2.8.0/debian/tests/control nix-2.16.1+dfsg/debian/tests/control --- nix-2.8.0/debian/tests/control 2022-05-02 18:55:23.000000000 +0000 +++ nix-2.16.1+dfsg/debian/tests/control 2023-10-04 02:33:09.000000000 +0000 @@ -2,16 +2,16 @@ Depends: @, make, systemd Restrictions: isolation-container, needs-root -Tests: nix-env -Depends: @, ca-certificates, make -Restrictions: allow-stderr, isolation-container, needs-root +#Tests: nix-env +#Depends: @, ca-certificates, make +#Restrictions: allow-stderr, isolation-container, needs-root, needs-internet, flaky # armhf: error: Package ‘hello-2.12’ in /nix/store/.../hello/default.nix:32 is not supported on ‘arm-linux’, refusing to evaluate. # ppc64el: error: evaluation aborted with the following error message: 'unsupported platform for the pure Linux stdenv' -Architecture: amd64 arm64 i386 +#Architecture: amd64 arm64 i386 -Tests: nix-shell -Depends: @, ca-certificates, make -Restrictions: allow-stderr, isolation-container, needs-root +#Tests: nix-shell +#Depends: @, ca-certificates, make +#Restrictions: allow-stderr, isolation-container, needs-root, needs-internet, flaky # armhf, ppc64el gives: # error: Package ‘shell’ in /nix/store/.../trivial-builders.nix:73 is not supported on ‘arm-linux’, refusing to evaluate. -Architecture: amd64 arm64 i386 \ No newline at end of file +#Architecture: amd64 arm64 i386 diff -Nru nix-2.8.0/debian/watch nix-2.16.1+dfsg/debian/watch --- nix-2.8.0/debian/watch 2022-05-02 18:55:23.000000000 +0000 +++ nix-2.16.1+dfsg/debian/watch 2023-10-04 02:33:09.000000000 +0000 @@ -1,8 +1,8 @@ version=4 -opts="mode=LWP,pgpmode=auto,pagemangle=s%nix/nix-([\.\d]+)/%$1%g,dversionmangle=s/\+dfsg\d*$//,repacksuffix=+dfsg1" \ -https://nix-releases.s3.amazonaws.com/?delimiter=/&prefix=nix/ \ -https://releases.nixos.org/nix/nix-[\.\d]+/nix-([\.\d]+).tar.xz - - - +opts=\ +repacksuffix=+dfsg,\ +dversionmangle=s/\+(debian|dfsg|ds|deb)(\.\d+)?$//,\ +repack,compression=xz,\ +filenamemangle=s/.+\/(\d+(?:\.\d+)+)\.tar\.gz/nix-$1\.tar\.gz/ \ +https://github.com/NixOS/nix/tags .*/(\d+(?:\.\d+)+)\.tar\.gz diff -Nru nix-2.8.0/default.nix nix-2.16.1+dfsg/default.nix --- nix-2.8.0/default.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/default.nix 2023-06-06 10:55:23.000000000 +0000 @@ -1,3 +1,10 @@ -(import (fetchTarball "https://github.com/edolstra/flake-compat/archive/master.tar.gz") { - src = ./.; -}).defaultNix +(import + ( + let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; + sha256 = lock.nodes.flake-compat.locked.narHash; + } + ) + { src = ./.; } +).defaultNix diff -Nru nix-2.8.0/doc/internal-api/doxygen.cfg.in nix-2.16.1+dfsg/doc/internal-api/doxygen.cfg.in --- nix-2.8.0/doc/internal-api/doxygen.cfg.in 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/internal-api/doxygen.cfg.in 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,63 @@ +# Doxyfile 1.9.5 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Nix" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = @PACKAGE_VERSION@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "Nix, the purely functional package manager; unstable internal interfaces" + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = NO + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +# FIXME Make this list more maintainable somehow. We could maybe generate this +# in the Makefile, but we would need to change how `.in` files are preprocessed +# so they can expand variables despite configure variables. + +INPUT = \ + src/libcmd \ + src/libexpr \ + src/libexpr/flake \ + src/libexpr/tests \ + src/libexpr/tests/value \ + src/libexpr/value \ + src/libfetchers \ + src/libmain \ + src/libstore \ + src/libstore/build \ + src/libstore/builtins \ + src/libstore/tests \ + src/libutil \ + src/libutil/tests \ + src/nix \ + src/nix-env \ + src/nix-store + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +INCLUDE_PATH = @RAPIDCHECK_HEADERS@ diff -Nru nix-2.8.0/doc/internal-api/.gitignore nix-2.16.1+dfsg/doc/internal-api/.gitignore --- nix-2.8.0/doc/internal-api/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/internal-api/.gitignore 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,3 @@ +/doxygen.cfg +/html +/latex diff -Nru nix-2.8.0/doc/internal-api/local.mk nix-2.16.1+dfsg/doc/internal-api/local.mk --- nix-2.8.0/doc/internal-api/local.mk 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/internal-api/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,19 @@ +.PHONY: internal-api-html + +ifeq ($(internal_api_docs), yes) + +$(docdir)/internal-api/html/index.html $(docdir)/internal-api/latex: $(d)/doxygen.cfg + mkdir -p $(docdir)/internal-api + { cat $< ; echo "OUTPUT_DIRECTORY=$(docdir)/internal-api" ; } | doxygen - + +# Generate the HTML API docs for Nix's unstable internal interfaces. +internal-api-html: $(docdir)/internal-api/html/index.html + +else + +# Make a nicer error message +internal-api-html: + @echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'." + @exit 1 + +endif diff -Nru nix-2.8.0/doc/manual/anchors.jq nix-2.16.1+dfsg/doc/manual/anchors.jq --- nix-2.8.0/doc/manual/anchors.jq 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/anchors.jq 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,31 @@ +"\\[\\]\\{#(?[^\\}]+?)\\}" as $empty_anchor_regex | +"\\[(?[^\\]]+?)\\]\\{#(?[^\\}]+?)\\}" as $anchor_regex | + + +def transform_anchors_html: + . | gsub($empty_anchor_regex; "") + | gsub($anchor_regex; "" + .text + ""); + + +def transform_anchors_strip: + . | gsub($empty_anchor_regex; "") + | gsub($anchor_regex; .text); + + +def map_contents_recursively(transformer): + . + { + Chapter: (.Chapter + { + content: .Chapter.content | transformer, + sub_items: .Chapter.sub_items | map(map_contents_recursively(transformer)), + }), + }; + + +def process_command: + .[0] as $context | + .[1] as $body | + $body + { + sections: $body.sections | map(map_contents_recursively(if $context.renderer == "html" then transform_anchors_html else transform_anchors_strip end)), + }; + +process_command diff -Nru nix-2.8.0/doc/manual/book.toml nix-2.16.1+dfsg/doc/manual/book.toml --- nix-2.8.0/doc/manual/book.toml 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/book.toml 2023-06-06 10:55:23.000000000 +0000 @@ -1,2 +1,21 @@ +[book] +title = "Nix Reference Manual" + [output.html] additional-css = ["custom.css"] +additional-js = ["redirects.js"] +edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}" +git-repository-url = "https://github.com/NixOS/nix" + +[preprocessor.anchors] +renderers = ["html"] +command = "jq --from-file doc/manual/anchors.jq" + +[output.linkcheck] +# no Internet during the build (in the sandbox) +follow-web-links = false + +# mdbook-linkcheck does not understand [foo]{#bar} style links, resulting in +# excessive "Potential incomplete link" warnings. No other kind of warning was +# produced at the time of writing. +warning-policy = "ignore" diff -Nru nix-2.8.0/doc/manual/custom.css nix-2.16.1+dfsg/doc/manual/custom.css --- nix-2.8.0/doc/manual/custom.css 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/custom.css 2023-06-06 10:55:23.000000000 +0000 @@ -5,3 +5,7 @@ h2 { margin-top: 1em; } + +.hljs-meta { + user-select: none; +} diff -Nru nix-2.8.0/doc/manual/generate-builtins.nix nix-2.16.1+dfsg/doc/manual/generate-builtins.nix --- nix-2.8.0/doc/manual/generate-builtins.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/generate-builtins.nix 2023-06-06 10:55:23.000000000 +0000 @@ -1,16 +1,24 @@ -with builtins; -with import ./utils.nix; +let + inherit (builtins) concatStringsSep attrNames; +in -builtins: +builtinsInfo: +let + showBuiltin = name: + let + inherit (builtinsInfo.${name}) doc args; + in + '' +
+ ${name} ${listArgs args} +
+
+ + ${doc} + +
+ ''; + listArgs = args: concatStringsSep " " (map (s: "${s}") args); +in +concatStringsSep "\n" (map showBuiltin (attrNames builtinsInfo)) -concatStrings (map - (name: - let builtin = builtins.${name}; in - "
${name} " - + concatStringsSep " " (map (s: "${s}") builtin.args) - + "
" - + "
\n\n" - + builtin.doc - + "\n\n
" - ) - (attrNames builtins)) diff -Nru nix-2.8.0/doc/manual/generate-manpage.nix nix-2.16.1+dfsg/doc/manual/generate-manpage.nix --- nix-2.8.0/doc/manual/generate-manpage.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/generate-manpage.nix 2023-06-06 10:55:23.000000000 +0000 @@ -1,99 +1,152 @@ -{ command, renderLinks ? false }: +let + inherit (builtins) + attrNames attrValues fromJSON listToAttrs mapAttrs + concatStringsSep concatMap length lessThan replaceStrings sort; + inherit (import ./utils.nix) concatStrings optionalString filterAttrs trim squash unique showSettings; +in -with builtins; -with import ./utils.nix; +commandDump: let - showCommand = - { command, def, filename }: - '' - **Warning**: This program is **experimental** and its interface is subject to change. - '' - + "# Name\n\n" - + "`${command}` - ${def.description}\n\n" - + "# Synopsis\n\n" - + showSynopsis { inherit command; args = def.args; } - + (if def.commands or {} != {} - then - let - categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues def.commands))); - listCommands = cmds: - concatStrings (map (name: - "* " - + (if renderLinks - then "[`${command} ${name}`](./${appendName filename name}.md)" - else "`${command} ${name}`") - + " - ${cmds.${name}.description}\n") - (attrNames cmds)); - in - "where *subcommand* is one of the following:\n\n" - # FIXME: group by category - + (if length categories > 1 - then - concatStrings (map - (cat: - "**${toString cat.description}:**\n\n" - + listCommands (filterAttrs (n: v: v.category == cat) def.commands) - + "\n" - ) categories) - + "\n" - else - listCommands def.commands - + "\n") - else "") - + (if def ? doc - then def.doc + "\n\n" - else "") - + (let s = showOptions def.flags; in - if s != "" - then "# Options\n\n${s}" - else "") - ; + commandInfo = fromJSON commandDump; + + showCommand = { command, details, filename, toplevel }: + let + + result = '' + > **Warning** \ + > This program is + > [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) + > and its interface is subject to change. + + # Name + + `${command}` - ${details.description} + + # Synopsis + + ${showSynopsis command details.args} + + ${maybeSubcommands} + + ${maybeDocumentation} + + ${maybeOptions} + ''; + + showSynopsis = command: args: + let + showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "..."; + arguments = concatStringsSep " " (map showArgument args); + in '' + `${command}` [*option*...] ${arguments} + ''; + + maybeSubcommands = optionalString (details ? commands && details.commands != {}) + '' + where *subcommand* is one of the following: + + ${subcommands} + ''; + + subcommands = if length categories > 1 + then listCategories + else listSubcommands details.commands; + + categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues details.commands))); + + listCategories = concatStrings (map showCategory categories); + + showCategory = cat: '' + **${toString cat.description}:** + + ${listSubcommands (filterAttrs (n: v: v.category == cat) details.commands)} + ''; + + listSubcommands = cmds: concatStrings (attrValues (mapAttrs showSubcommand cmds)); + + showSubcommand = name: subcmd: '' + * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} + ''; + + maybeDocumentation = optionalString + (details ? doc) + (replaceStrings ["@stores@"] [storeDocs] details.doc); + + maybeOptions = optionalString (details.flags != {}) '' + # Options + + ${showOptions details.flags toplevel.flags} + ''; + + showOptions = options: commonOptions: + let + allOptions = options // commonOptions; + showCategory = cat: '' + ${optionalString (cat != "") "**${cat}:**"} + + ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)} + ''; + listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts)); + showOption = name: option: + let + shortName = optionalString + (option ? shortName) + ("/ `-${option.shortName}`"); + labels = optionalString + (option ? labels) + (concatStringsSep " " (map (s: "*${s}*") option.labels)); + in trim '' + - `--${name}` ${shortName} ${labels} + + ${option.description} + ''; + categories = sort lessThan (unique (map (cmd: cmd.category) (attrValues allOptions))); + in concatStrings (map showCategory categories); + in squash result; appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name; - showOptions = flags: + processCommand = { command, details, filename, toplevel }: let - categories = sort builtins.lessThan (unique (map (cmd: cmd.category) (attrValues flags))); - in - concatStrings (map - (cat: - (if cat != "" - then "**${cat}:**\n\n" - else "") - + concatStrings - (map (longName: - let - flag = flags.${longName}; - in - " - `--${longName}`" - + (if flag ? shortName then " / `-${flag.shortName}`" else "") - + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "") - + " \n" - + " " + flag.description + "\n\n" - ) (attrNames (filterAttrs (n: v: v.category == cat) flags)))) - categories); - - showSynopsis = - { command, args }: - "`${command}` [*option*...] ${concatStringsSep " " - (map (arg: "*${arg.label}*" + (if arg ? arity then "" else "...")) args)}\n\n"; - - processCommand = { command, def, filename }: - [ { name = filename + ".md"; value = showCommand { inherit command def filename; }; inherit command; } ] - ++ concatMap - (name: processCommand { - filename = appendName filename name; - command = command + " " + name; - def = def.commands.${name}; - }) - (attrNames def.commands or {}); + cmd = { + inherit command; + name = filename + ".md"; + value = showCommand { inherit command details filename toplevel; }; + }; + subcommand = subCmd: processCommand { + command = command + " " + subCmd; + details = details.commands.${subCmd}; + filename = appendName filename subCmd; + inherit toplevel; + }; + in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {}); + + manpages = processCommand { + command = "nix"; + details = commandInfo.args; + filename = "nix"; + toplevel = commandInfo.args; + }; + + tableOfContents = let + showEntry = page: + " - [${page.command}](command-ref/new-cli/${page.name})"; + in concatStringsSep "\n" (map showEntry manpages) + "\n"; -in + storeDocs = + let + showStore = name: { settings, doc }: + '' + ## ${name} -let - manpages = processCommand { filename = "nix"; command = "nix"; def = builtins.fromJSON command; }; - summary = concatStrings (map (manpage: " - [${manpage.command}](command-ref/new-cli/${manpage.name})\n") manpages); -in -(listToAttrs manpages) // { "SUMMARY.md" = summary; } + ${doc} + + **Settings**: + + ${showSettings { useAnchors = false; } settings} + ''; + in concatStrings (attrValues (mapAttrs showStore commandInfo.stores)); + +in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; } diff -Nru nix-2.8.0/doc/manual/generate-options.nix nix-2.16.1+dfsg/doc/manual/generate-options.nix --- nix-2.8.0/doc/manual/generate-options.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/generate-options.nix 1970-01-01 00:00:00.000000000 +0000 @@ -1,29 +0,0 @@ -with builtins; -with import ./utils.nix; - -options: - -concatStrings (map - (name: - let option = options.${name}; in - " - [`${name}`](#conf-${name})" - + "

\n\n" - + concatStrings (map (s: " ${s}\n") (splitLines option.description)) + "\n\n" - + (if option.documentDefault - then " **Default:** " + ( - if option.value == "" || option.value == [] - then "*empty*" - else if isBool option.value - then (if option.value then "`true`" else "`false`") - else - # n.b. a StringMap value type is specified as a string, but - # this shows the value type. The empty stringmap is "null" in - # JSON, but that converts to "{ }" here. - (if isAttrs option.value then "`\"\"`" - else "`" + toString option.value + "`")) + "\n\n" - else " **Default:** *machine-specific*\n") - + (if option.aliases != [] - then " **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n" - else "") - ) - (attrNames options)) diff -Nru nix-2.8.0/doc/manual/generate-xp-features.nix nix-2.16.1+dfsg/doc/manual/generate-xp-features.nix --- nix-2.8.0/doc/manual/generate-xp-features.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/generate-xp-features.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,11 @@ +with builtins; +with import ./utils.nix; + +let + showExperimentalFeature = name: doc: + squash '' + ## [`${name}`]{#xp-feature-${name}} + + ${doc} + ''; +in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) diff -Nru nix-2.8.0/doc/manual/generate-xp-features-shortlist.nix nix-2.16.1+dfsg/doc/manual/generate-xp-features-shortlist.nix --- nix-2.8.0/doc/manual/generate-xp-features-shortlist.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/generate-xp-features-shortlist.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,9 @@ +with builtins; +with import ./utils.nix; + +let + showExperimentalFeature = name: doc: + '' + - [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name}) + ''; +in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff -Nru nix-2.8.0/doc/manual/local.mk nix-2.16.1+dfsg/doc/manual/local.mk --- nix-2.8.0/doc/manual/local.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -1,13 +1,25 @@ ifeq ($(doc_generate),yes) -# Generate man pages. +MANUAL_SRCS := \ + $(call rwildcard, $(d)/src, *.md) \ + $(call rwildcard, $(d)/src, */*.md) + man-pages := $(foreach n, \ - nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ - nix-collect-garbage.1 \ - nix-prefetch-url.1 nix-channel.1 \ - nix-hash.1 nix-copy-closure.1 \ - nix.conf.5 nix-daemon.8, \ - $(d)/$(n)) + nix-env.1 nix-store.1 \ + nix-build.1 nix-shell.1 nix-instantiate.1 \ + nix-collect-garbage.1 \ + nix-prefetch-url.1 nix-channel.1 \ + nix-hash.1 nix-copy-closure.1 \ + nix.conf.5 nix-daemon.8 \ + nix-profiles.5 \ +, $(d)/$(n)) + +# man pages for subcommands +# convert from `$(d)/src/command-ref/nix-{1}/{2}.md` to `$(d)/nix-{1}-{2}.1` +# FIXME: unify with how nix3-cli man pages are generated +man-pages += $(foreach subcommand, \ + $(filter-out %opt-common.md %env-common.md, $(wildcard $(d)/src/command-ref/nix-*/*.md)), \ + $(d)/$(subst /,-,$(subst $(d)/src/command-ref/,,$(subst .md,.1,$(subcommand))))) clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 @@ -22,49 +34,104 @@ nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw +# re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution +define process-includes + while read -r line; do \ + set -euo pipefail; \ + filename="$$(dirname $(1))/$$(sed 's/{{#include \(.*\)}}/\1/'<<< $$line)"; \ + test -f "$$filename" || ( echo "#include-d file '$$filename' does not exist." >&2; exit 1; ); \ + matchline="$$(sed 's|/|\\/|g' <<< $$line)"; \ + sed -i "/$$matchline/r $$filename" $(2); \ + sed -i "s/$$matchline//" $(2); \ + done < <(grep '{{#include' $(1)) +endef + +$(d)/nix-env-%.1: $(d)/src/command-ref/nix-env/%.md + @printf "Title: %s\n\n" "$(subst nix-env-,nix-env --,$$(basename "$@" .1))" > $^.tmp + $(render-subcommand) + +$(d)/nix-store-%.1: $(d)/src/command-ref/nix-store/%.md + @printf -- 'Title: %s\n\n' "$(subst nix-store-,nix-store --,$$(basename "$@" .1))" > $^.tmp + $(render-subcommand) + +# FIXME: there surely is some more deduplication to be achieved here with even darker Make magic +define render-subcommand + @cat $^ >> $^.tmp + @$(call process-includes,$^,$^.tmp) + $(trace-gen) lowdown -sT man --nroff-nolinks -M section=1 $^.tmp -o $@ + @# fix up `lowdown`'s automatic escaping of `--` + @# https://github.com/kristapsdz/lowdown/blob/edca6ce6d5336efb147321a43c47a698de41bb7c/entity.c#L202 + @sed -i 's/\e\[u2013\]/--/' $@ + @rm $^.tmp +endef + + $(d)/%.1: $(d)/src/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp @cat $^ >> $^.tmp - $(trace-gen) lowdown -sT man -M section=1 $^.tmp -o $@ + @$(call process-includes,$^,$^.tmp) + $(trace-gen) lowdown -sT man --nroff-nolinks -M section=1 $^.tmp -o $@ @rm $^.tmp $(d)/%.8: $(d)/src/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .8)" > $^.tmp @cat $^ >> $^.tmp - $(trace-gen) lowdown -sT man -M section=8 $^.tmp -o $@ + $(trace-gen) lowdown -sT man --nroff-nolinks -M section=8 $^.tmp -o $@ @rm $^.tmp $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp @cat $^ >> $^.tmp - $(trace-gen) lowdown -sT man -M section=5 $^.tmp -o $@ + @$(call process-includes,$^,$^.tmp) + $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli - $(trace-gen) cat doc/manual/src/SUMMARY.md.in | while IFS= read line; do if [[ $$line = @manpages@ ]]; then cat doc/manual/src/command-ref/new-cli/SUMMARY.md; else echo "$$line"; fi; done > $@.tmp - @mv $@.tmp $@ +$(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md + @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp + @cat $^ >> $^.tmp + $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ + @rm $^.tmp -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix - @rm -rf $@ - $(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { command = builtins.readFile $<; renderLinks = true; }' +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md + @cp $< $@ + @$(call process-includes,$@,$@) -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' + @mv $@.tmp $@ + +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-args > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp @mv $@.tmp $@ $(d)/conf-file.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix - @cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp - @cat doc/manual/src/expressions/builtins-suffix.md >> $@.tmp +$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + +$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + +$(d)/xp-features.json: $(bindir)/nix + $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp + @mv $@.tmp $@ + +$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix + @cat doc/manual/src/language/builtins-prefix.md > $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ $(d)/builtins.json: $(bindir)/nix @@ -72,7 +139,8 @@ @mv $@.tmp $@ # Generate the HTML manual. -html: $(docdir)/manual/index.html +.PHONY: manual-html +manual-html: $(docdir)/manual/index.html install: $(docdir)/manual/index.html # Generate 'nix' manpages. @@ -80,6 +148,8 @@ man: doc/manual/generated/man1/nix3-manpages all: doc/manual/generated/man1/nix3-manpages +# FIXME: unify with how the other man pages are generated. +# this one works differently and does not use any of the amenities provided by `/mk/lib.mk`. $(mandir)/man1/nix3-manpages: doc/manual/generated/man1/nix3-manpages @mkdir -p $(DESTDIR)$$(dirname $@) $(trace-install) install -m 0644 $$(dirname $<)/* $(DESTDIR)$$(dirname $@) @@ -87,17 +157,33 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli @mkdir -p $(DESTDIR)$$(dirname $@) $(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \ - name=$$(basename $$i .md); \ - tmpFile=$$(mktemp); \ - if [[ $$name = SUMMARY ]]; then continue; fi; \ - printf "Title: %s\n\n" "$$name" > $$tmpFile; \ - cat $$i >> $$tmpFile; \ - lowdown -sT man -M section=1 $$tmpFile -o $(DESTDIR)$$(dirname $@)/$$name.1; \ - rm $$tmpFile; \ + name=$$(basename $$i .md); \ + tmpFile=$$(mktemp); \ + if [[ $$name = SUMMARY ]]; then continue; fi; \ + printf "Title: %s\n\n" "$$name" > $$tmpFile; \ + cat $$i >> $$tmpFile; \ + lowdown -sT man --nroff-nolinks -M section=1 $$tmpFile -o $(DESTDIR)$$(dirname $@)/$$name.1; \ + rm $$tmpFile; \ done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(call rwildcard, $(d)/src, *.md) - $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md + $(trace-gen) \ + tmp="$$(mktemp -d)"; \ + cp -r doc/manual "$$tmp"; \ + find "$$tmp" -name '*.md' | while read -r file; do \ + $(call process-includes,$$file,$$file); \ + done; \ + find "$$tmp" -name '*.md' | while read -r file; do \ + docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/src)"; \ + sed -i "s,@docroot@,$$docroot,g" "$$file"; \ + done; \ + set -euo pipefail; \ + RUST_LOG=warn mdbook build "$$tmp/manual" -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \ + | { grep -Fv "because fragment resolution isn't implemented" || :; }; \ + rm -rf "$$tmp/manual" + @rm -rf $(DESTDIR)$(docdir)/manual + @mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual + @rm -rf $(DESTDIR)$(docdir)/manual.tmp endif diff -Nru nix-2.8.0/doc/manual/redirects.js nix-2.16.1+dfsg/doc/manual/redirects.js --- nix-2.8.0/doc/manual/redirects.js 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/redirects.js 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,424 @@ +// redirect rules for anchors ensure backwards compatibility of URLs. +// this must be done on the client side, as web servers do not see the anchor part of the URL. + +// redirections are declared as follows: +// each entry has as its key a path matching the requested URL path, relative to the mdBook document root. +// +// IMPORTANT: it must specify the full path with file name and suffix +// +// each entry is itself a set of key-value pairs, where +// - keys are anchors on the matched path. +// - values are redirection targets relative to the current path. + +const redirects = { + "index.html": { + "part-advanced-topics": "advanced-topics/advanced-topics.html", + "chap-tuning-cores-and-jobs": "advanced-topics/cores-vs-jobs.html", + "chap-diff-hook": "advanced-topics/diff-hook.html", + "check-dirs-are-unregistered": "advanced-topics/diff-hook.html#check-dirs-are-unregistered", + "chap-distributed-builds": "advanced-topics/distributed-builds.html", + "chap-post-build-hook": "advanced-topics/post-build-hook.html", + "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats", + "part-command-ref": "command-ref/command-ref.html", + "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation", + "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges", + "conf-allowed-uris": "command-ref/conf-file.html#conf-allowed-uris", + "conf-allowed-users": "command-ref/conf-file.html#conf-allowed-users", + "conf-auto-optimise-store": "command-ref/conf-file.html#conf-auto-optimise-store", + "conf-binary-cache-public-keys": "command-ref/conf-file.html#conf-binary-cache-public-keys", + "conf-binary-caches": "command-ref/conf-file.html#conf-binary-caches", + "conf-build-compress-log": "command-ref/conf-file.html#conf-build-compress-log", + "conf-build-cores": "command-ref/conf-file.html#conf-build-cores", + "conf-build-extra-chroot-dirs": "command-ref/conf-file.html#conf-build-extra-chroot-dirs", + "conf-build-extra-sandbox-paths": "command-ref/conf-file.html#conf-build-extra-sandbox-paths", + "conf-build-fallback": "command-ref/conf-file.html#conf-build-fallback", + "conf-build-max-jobs": "command-ref/conf-file.html#conf-build-max-jobs", + "conf-build-max-log-size": "command-ref/conf-file.html#conf-build-max-log-size", + "conf-build-max-silent-time": "command-ref/conf-file.html#conf-build-max-silent-time", + "conf-build-timeout": "command-ref/conf-file.html#conf-build-timeout", + "conf-build-use-chroot": "command-ref/conf-file.html#conf-build-use-chroot", + "conf-build-use-sandbox": "command-ref/conf-file.html#conf-build-use-sandbox", + "conf-build-use-substitutes": "command-ref/conf-file.html#conf-build-use-substitutes", + "conf-build-users-group": "command-ref/conf-file.html#conf-build-users-group", + "conf-builders": "command-ref/conf-file.html#conf-builders", + "conf-builders-use-substitutes": "command-ref/conf-file.html#conf-builders-use-substitutes", + "conf-compress-build-log": "command-ref/conf-file.html#conf-compress-build-log", + "conf-connect-timeout": "command-ref/conf-file.html#conf-connect-timeout", + "conf-cores": "command-ref/conf-file.html#conf-cores", + "conf-diff-hook": "command-ref/conf-file.html#conf-diff-hook", + "conf-env-keep-derivations": "command-ref/conf-file.html#conf-env-keep-derivations", + "conf-extra-binary-caches": "command-ref/conf-file.html#conf-extra-binary-caches", + "conf-extra-platforms": "command-ref/conf-file.html#conf-extra-platforms", + "conf-extra-sandbox-paths": "command-ref/conf-file.html#conf-extra-sandbox-paths", + "conf-extra-substituters": "command-ref/conf-file.html#conf-extra-substituters", + "conf-fallback": "command-ref/conf-file.html#conf-fallback", + "conf-fsync-metadata": "command-ref/conf-file.html#conf-fsync-metadata", + "conf-gc-keep-derivations": "command-ref/conf-file.html#conf-gc-keep-derivations", + "conf-gc-keep-outputs": "command-ref/conf-file.html#conf-gc-keep-outputs", + "conf-hashed-mirrors": "command-ref/conf-file.html#conf-hashed-mirrors", + "conf-http-connections": "command-ref/conf-file.html#conf-http-connections", + "conf-keep-build-log": "command-ref/conf-file.html#conf-keep-build-log", + "conf-keep-derivations": "command-ref/conf-file.html#conf-keep-derivations", + "conf-keep-env-derivations": "command-ref/conf-file.html#conf-keep-env-derivations", + "conf-keep-outputs": "command-ref/conf-file.html#conf-keep-outputs", + "conf-max-build-log-size": "command-ref/conf-file.html#conf-max-build-log-size", + "conf-max-free": "command-ref/conf-file.html#conf-max-free", + "conf-max-jobs": "command-ref/conf-file.html#conf-max-jobs", + "conf-max-silent-time": "command-ref/conf-file.html#conf-max-silent-time", + "conf-min-free": "command-ref/conf-file.html#conf-min-free", + "conf-narinfo-cache-negative-ttl": "command-ref/conf-file.html#conf-narinfo-cache-negative-ttl", + "conf-narinfo-cache-positive-ttl": "command-ref/conf-file.html#conf-narinfo-cache-positive-ttl", + "conf-netrc-file": "command-ref/conf-file.html#conf-netrc-file", + "conf-plugin-files": "command-ref/conf-file.html#conf-plugin-files", + "conf-post-build-hook": "command-ref/conf-file.html#conf-post-build-hook", + "conf-pre-build-hook": "command-ref/conf-file.html#conf-pre-build-hook", + "conf-require-sigs": "command-ref/conf-file.html#conf-require-sigs", + "conf-restrict-eval": "command-ref/conf-file.html#conf-restrict-eval", + "conf-run-diff-hook": "command-ref/conf-file.html#conf-run-diff-hook", + "conf-sandbox": "command-ref/conf-file.html#conf-sandbox", + "conf-sandbox-dev-shm-size": "command-ref/conf-file.html#conf-sandbox-dev-shm-size", + "conf-sandbox-paths": "command-ref/conf-file.html#conf-sandbox-paths", + "conf-secret-key-files": "command-ref/conf-file.html#conf-secret-key-files", + "conf-show-trace": "command-ref/conf-file.html#conf-show-trace", + "conf-stalled-download-timeout": "command-ref/conf-file.html#conf-stalled-download-timeout", + "conf-substitute": "command-ref/conf-file.html#conf-substitute", + "conf-substituters": "command-ref/conf-file.html#conf-substituters", + "conf-system": "command-ref/conf-file.html#conf-system", + "conf-system-features": "command-ref/conf-file.html#conf-system-features", + "conf-tarball-ttl": "command-ref/conf-file.html#conf-tarball-ttl", + "conf-timeout": "command-ref/conf-file.html#conf-timeout", + "conf-trace-function-calls": "command-ref/conf-file.html#conf-trace-function-calls", + "conf-trusted-binary-caches": "command-ref/conf-file.html#conf-trusted-binary-caches", + "conf-trusted-public-keys": "command-ref/conf-file.html#conf-trusted-public-keys", + "conf-trusted-substituters": "command-ref/conf-file.html#conf-trusted-substituters", + "conf-trusted-users": "command-ref/conf-file.html#conf-trusted-users", + "extra-sandbox-paths": "command-ref/conf-file.html#extra-sandbox-paths", + "sec-conf-file": "command-ref/conf-file.html", + "env-NIX_PATH": "command-ref/env-common.html#env-NIX_PATH", + "env-common": "command-ref/env-common.html", + "envar-remote": "command-ref/env-common.html#env-NIX_REMOTE", + "sec-common-env": "command-ref/env-common.html", + "ch-files": "command-ref/files.html", + "ch-main-commands": "command-ref/main-commands.html", + "opt-out-link": "command-ref/nix-build.html#opt-out-link", + "sec-nix-build": "command-ref/nix-build.html", + "sec-nix-channel": "command-ref/nix-channel.html", + "sec-nix-collect-garbage": "command-ref/nix-collect-garbage.html", + "sec-nix-copy-closure": "command-ref/nix-copy-closure.html", + "sec-nix-daemon": "command-ref/nix-daemon.html", + "refsec-nix-env-install-examples": "command-ref/nix-env.html#examples", + "rsec-nix-env-install": "command-ref/nix-env.html#operation---install", + "rsec-nix-env-set": "command-ref/nix-env.html#operation---set", + "rsec-nix-env-set-flag": "command-ref/nix-env.html#operation---set-flag", + "rsec-nix-env-upgrade": "command-ref/nix-env.html#operation---upgrade", + "sec-nix-env": "command-ref/nix-env.html", + "ssec-version-comparisons": "command-ref/nix-env.html#versions", + "sec-nix-hash": "command-ref/nix-hash.html", + "sec-nix-instantiate": "command-ref/nix-instantiate.html", + "sec-nix-prefetch-url": "command-ref/nix-prefetch-url.html", + "sec-nix-shell": "command-ref/nix-shell.html", + "ssec-nix-shell-shebang": "command-ref/nix-shell.html#use-as-a--interpreter", + "nixref-queries": "command-ref/nix-store.html#queries", + "opt-add-root": "command-ref/nix-store.html#opt-add-root", + "refsec-nix-store-dump": "command-ref/nix-store.html#operation---dump", + "refsec-nix-store-export": "command-ref/nix-store.html#operation---export", + "refsec-nix-store-import": "command-ref/nix-store.html#operation---import", + "refsec-nix-store-query": "command-ref/nix-store.html#operation---query", + "refsec-nix-store-verify": "command-ref/nix-store.html#operation---verify", + "rsec-nix-store-gc": "command-ref/nix-store.html#operation---gc", + "rsec-nix-store-generate-binary-cache-key": "command-ref/nix-store.html#operation---generate-binary-cache-key", + "rsec-nix-store-realise": "command-ref/nix-store.html#operation---realise", + "rsec-nix-store-serve": "command-ref/nix-store.html#operation---serve", + "sec-nix-store": "command-ref/nix-store.html", + "opt-I": "command-ref/opt-common.html#opt-I", + "opt-attr": "command-ref/opt-common.html#opt-attr", + "opt-common": "command-ref/opt-common.html", + "opt-cores": "command-ref/opt-common.html#opt-cores", + "opt-log-format": "command-ref/opt-common.html#opt-log-format", + "opt-max-jobs": "command-ref/opt-common.html#opt-max-jobs", + "opt-max-silent-time": "command-ref/opt-common.html#opt-max-silent-time", + "opt-timeout": "command-ref/opt-common.html#opt-timeout", + "sec-common-options": "command-ref/opt-common.html", + "ch-utilities": "command-ref/utilities.html", + "chap-hacking": "contributing/hacking.html", + "adv-attr-allowSubstitutes": "language/advanced-attributes.html#adv-attr-allowSubstitutes", + "adv-attr-allowedReferences": "language/advanced-attributes.html#adv-attr-allowedReferences", + "adv-attr-allowedRequisites": "language/advanced-attributes.html#adv-attr-allowedRequisites", + "adv-attr-disallowedReferences": "language/advanced-attributes.html#adv-attr-disallowedReferences", + "adv-attr-disallowedRequisites": "language/advanced-attributes.html#adv-attr-disallowedRequisites", + "adv-attr-exportReferencesGraph": "language/advanced-attributes.html#adv-attr-exportReferencesGraph", + "adv-attr-impureEnvVars": "language/advanced-attributes.html#adv-attr-impureEnvVars", + "adv-attr-outputHash": "language/advanced-attributes.html#adv-attr-outputHash", + "adv-attr-outputHashAlgo": "language/advanced-attributes.html#adv-attr-outputHashAlgo", + "adv-attr-outputHashMode": "language/advanced-attributes.html#adv-attr-outputHashMode", + "adv-attr-passAsFile": "language/advanced-attributes.html#adv-attr-passAsFile", + "adv-attr-preferLocalBuild": "language/advanced-attributes.html#adv-attr-preferLocalBuild", + "fixed-output-drvs": "language/advanced-attributes.html#adv-attr-outputHash", + "sec-advanced-attributes": "language/advanced-attributes.html", + "builtin-abort": "language/builtins.html#builtins-abort", + "builtin-add": "language/builtins.html#builtins-add", + "builtin-all": "language/builtins.html#builtins-all", + "builtin-any": "language/builtins.html#builtins-any", + "builtin-attrNames": "language/builtins.html#builtins-attrNames", + "builtin-attrValues": "language/builtins.html#builtins-attrValues", + "builtin-baseNameOf": "language/builtins.html#builtins-baseNameOf", + "builtin-bitAnd": "language/builtins.html#builtins-bitAnd", + "builtin-bitOr": "language/builtins.html#builtins-bitOr", + "builtin-bitXor": "language/builtins.html#builtins-bitXor", + "builtin-builtins": "language/builtins.html#builtins-builtins", + "builtin-compareVersions": "language/builtins.html#builtins-compareVersions", + "builtin-concatLists": "language/builtins.html#builtins-concatLists", + "builtin-concatStringsSep": "language/builtins.html#builtins-concatStringsSep", + "builtin-currentSystem": "language/builtins.html#builtins-currentSystem", + "builtin-deepSeq": "language/builtins.html#builtins-deepSeq", + "builtin-derivation": "language/builtins.html#builtins-derivation", + "builtin-dirOf": "language/builtins.html#builtins-dirOf", + "builtin-div": "language/builtins.html#builtins-div", + "builtin-elem": "language/builtins.html#builtins-elem", + "builtin-elemAt": "language/builtins.html#builtins-elemAt", + "builtin-fetchGit": "language/builtins.html#builtins-fetchGit", + "builtin-fetchTarball": "language/builtins.html#builtins-fetchTarball", + "builtin-fetchurl": "language/builtins.html#builtins-fetchurl", + "builtin-filterSource": "language/builtins.html#builtins-filterSource", + "builtin-foldl-prime": "language/builtins.html#builtins-foldl-prime", + "builtin-fromJSON": "language/builtins.html#builtins-fromJSON", + "builtin-functionArgs": "language/builtins.html#builtins-functionArgs", + "builtin-genList": "language/builtins.html#builtins-genList", + "builtin-getAttr": "language/builtins.html#builtins-getAttr", + "builtin-getEnv": "language/builtins.html#builtins-getEnv", + "builtin-hasAttr": "language/builtins.html#builtins-hasAttr", + "builtin-hashFile": "language/builtins.html#builtins-hashFile", + "builtin-hashString": "language/builtins.html#builtins-hashString", + "builtin-head": "language/builtins.html#builtins-head", + "builtin-import": "language/builtins.html#builtins-import", + "builtin-intersectAttrs": "language/builtins.html#builtins-intersectAttrs", + "builtin-isAttrs": "language/builtins.html#builtins-isAttrs", + "builtin-isBool": "language/builtins.html#builtins-isBool", + "builtin-isFloat": "language/builtins.html#builtins-isFloat", + "builtin-isFunction": "language/builtins.html#builtins-isFunction", + "builtin-isInt": "language/builtins.html#builtins-isInt", + "builtin-isList": "language/builtins.html#builtins-isList", + "builtin-isNull": "language/builtins.html#builtins-isNull", + "builtin-isString": "language/builtins.html#builtins-isString", + "builtin-length": "language/builtins.html#builtins-length", + "builtin-lessThan": "language/builtins.html#builtins-lessThan", + "builtin-listToAttrs": "language/builtins.html#builtins-listToAttrs", + "builtin-map": "language/builtins.html#builtins-map", + "builtin-match": "language/builtins.html#builtins-match", + "builtin-mul": "language/builtins.html#builtins-mul", + "builtin-parseDrvName": "language/builtins.html#builtins-parseDrvName", + "builtin-path": "language/builtins.html#builtins-path", + "builtin-pathExists": "language/builtins.html#builtins-pathExists", + "builtin-placeholder": "language/builtins.html#builtins-placeholder", + "builtin-readDir": "language/builtins.html#builtins-readDir", + "builtin-readFile": "language/builtins.html#builtins-readFile", + "builtin-removeAttrs": "language/builtins.html#builtins-removeAttrs", + "builtin-replaceStrings": "language/builtins.html#builtins-replaceStrings", + "builtin-seq": "language/builtins.html#builtins-seq", + "builtin-sort": "language/builtins.html#builtins-sort", + "builtin-split": "language/builtins.html#builtins-split", + "builtin-splitVersion": "language/builtins.html#builtins-splitVersion", + "builtin-stringLength": "language/builtins.html#builtins-stringLength", + "builtin-sub": "language/builtins.html#builtins-sub", + "builtin-substring": "language/builtins.html#builtins-substring", + "builtin-tail": "language/builtins.html#builtins-tail", + "builtin-throw": "language/builtins.html#builtins-throw", + "builtin-toFile": "language/builtins.html#builtins-toFile", + "builtin-toJSON": "language/builtins.html#builtins-toJSON", + "builtin-toPath": "language/builtins.html#builtins-toPath", + "builtin-toString": "language/builtins.html#builtins-toString", + "builtin-toXML": "language/builtins.html#builtins-toXML", + "builtin-trace": "language/builtins.html#builtins-trace", + "builtin-tryEval": "language/builtins.html#builtins-tryEval", + "builtin-typeOf": "language/builtins.html#builtins-typeOf", + "ssec-builtins": "language/builtins.html", + "attr-system": "language/derivations.html#attr-system", + "ssec-derivation": "language/derivations.html", + "ch-expression-language": "language/index.html", + "sec-constructs": "language/constructs.html", + "sect-let-language": "language/constructs.html#let-language", + "ss-functions": "language/constructs.html#functions", + "sec-language-operators": "language/operators.html", + "table-operators": "language/operators.html", + "ssec-values": "language/values.html", + "gloss-closure": "glossary.html#gloss-closure", + "gloss-derivation": "glossary.html#gloss-derivation", + "gloss-deriver": "glossary.html#gloss-deriver", + "gloss-nar": "glossary.html#gloss-nar", + "gloss-output-path": "glossary.html#gloss-output-path", + "gloss-profile": "glossary.html#gloss-profile", + "gloss-reachable": "glossary.html#gloss-reachable", + "gloss-reference": "glossary.html#gloss-reference", + "gloss-substitute": "glossary.html#gloss-substitute", + "gloss-user-env": "glossary.html#gloss-user-env", + "gloss-validity": "glossary.html#gloss-validity", + "part-glossary": "glossary.html", + "sec-building-source": "installation/building-source.html", + "ch-env-variables": "installation/env-variables.html", + "sec-installer-proxy-settings": "installation/env-variables.html#proxy-environment-variables", + "sec-nix-ssl-cert-file": "installation/env-variables.html#nix_ssl_cert_file", + "sec-nix-ssl-cert-file-with-nix-daemon-and-macos": "installation/env-variables.html#nix_ssl_cert_file-with-macos-and-the-nix-daemon", + "chap-installation": "installation/installation.html", + "ch-installing-binary": "installation/installing-binary.html", + "sect-macos-installation": "installation/installing-binary.html#macos-installation", + "sect-macos-installation-change-store-prefix": "installation/installing-binary.html#macos-installation", + "sect-macos-installation-encrypted-volume": "installation/installing-binary.html#macos-installation", + "sect-macos-installation-recommended-notes": "installation/installing-binary.html#macos-installation", + "sect-macos-installation-symlink": "installation/installing-binary.html#macos-installation", + "sect-multi-user-installation": "installation/installing-binary.html#multi-user-installation", + "sect-nix-install-binary-tarball": "installation/installing-binary.html#installing-from-a-binary-tarball", + "sect-nix-install-pinned-version-url": "installation/installing-binary.html#installing-a-pinned-nix-version-from-a-url", + "sect-single-user-installation": "installation/installing-binary.html#single-user-installation", + "ch-installing-source": "installation/installing-source.html", + "ssec-multi-user": "installation/multi-user.html", + "ch-nix-security": "installation/nix-security.html", + "sec-obtaining-source": "installation/obtaining-source.html", + "sec-prerequisites-source": "installation/prerequisites-source.html", + "sec-single-user": "installation/single-user.html", + "ch-supported-platforms": "installation/supported-platforms.html", + "ch-upgrading-nix": "installation/upgrading.html", + "ch-about-nix": "introduction.html", + "chap-introduction": "introduction.html", + "ch-basic-package-mgmt": "package-management/basic-package-mgmt.html", + "ssec-binary-cache-substituter": "package-management/binary-cache-substituter.html", + "sec-channels": "package-management/channels.html", + "ssec-copy-closure": "package-management/copy-closure.html", + "sec-garbage-collection": "package-management/garbage-collection.html", + "ssec-gc-roots": "package-management/garbage-collector-roots.html", + "chap-package-management": "package-management/package-management.html", + "sec-profiles": "package-management/profiles.html", + "ssec-s3-substituter": "package-management/s3-substituter.html", + "ssec-s3-substituter-anonymous-reads": "package-management/s3-substituter.html#anonymous-reads-to-your-s3-compatible-binary-cache", + "ssec-s3-substituter-authenticated-reads": "package-management/s3-substituter.html#authenticated-reads-to-your-s3-binary-cache", + "ssec-s3-substituter-authenticated-writes": "package-management/s3-substituter.html#authenticated-writes-to-your-s3-compatible-binary-cache", + "sec-sharing-packages": "package-management/sharing-packages.html", + "ssec-ssh-substituter": "package-management/ssh-substituter.html", + "chap-quick-start": "quick-start.html", + "sec-relnotes": "release-notes/release-notes.html", + "ch-relnotes-0.10.1": "release-notes/rl-0.10.1.html", + "ch-relnotes-0.10": "release-notes/rl-0.10.html", + "ssec-relnotes-0.11": "release-notes/rl-0.11.html", + "ssec-relnotes-0.12": "release-notes/rl-0.12.html", + "ssec-relnotes-0.13": "release-notes/rl-0.13.html", + "ssec-relnotes-0.14": "release-notes/rl-0.14.html", + "ssec-relnotes-0.15": "release-notes/rl-0.15.html", + "ssec-relnotes-0.16": "release-notes/rl-0.16.html", + "ch-relnotes-0.5": "release-notes/rl-0.5.html", + "ch-relnotes-0.6": "release-notes/rl-0.6.html", + "ch-relnotes-0.7": "release-notes/rl-0.7.html", + "ch-relnotes-0.8.1": "release-notes/rl-0.8.1.html", + "ch-relnotes-0.8": "release-notes/rl-0.8.html", + "ch-relnotes-0.9.1": "release-notes/rl-0.9.1.html", + "ch-relnotes-0.9.2": "release-notes/rl-0.9.2.html", + "ch-relnotes-0.9": "release-notes/rl-0.9.html", + "ssec-relnotes-1.0": "release-notes/rl-1.0.html", + "ssec-relnotes-1.1": "release-notes/rl-1.1.html", + "ssec-relnotes-1.10": "release-notes/rl-1.10.html", + "ssec-relnotes-1.11.10": "release-notes/rl-1.11.10.html", + "ssec-relnotes-1.11": "release-notes/rl-1.11.html", + "ssec-relnotes-1.2": "release-notes/rl-1.2.html", + "ssec-relnotes-1.3": "release-notes/rl-1.3.html", + "ssec-relnotes-1.4": "release-notes/rl-1.4.html", + "ssec-relnotes-1.5.1": "release-notes/rl-1.5.1.html", + "ssec-relnotes-1.5.2": "release-notes/rl-1.5.2.html", + "ssec-relnotes-1.5": "release-notes/rl-1.5.html", + "ssec-relnotes-1.6.1": "release-notes/rl-1.6.1.html", + "ssec-relnotes-1.6.0": "release-notes/rl-1.6.html", + "ssec-relnotes-1.7": "release-notes/rl-1.7.html", + "ssec-relnotes-1.8": "release-notes/rl-1.8.html", + "ssec-relnotes-1.9": "release-notes/rl-1.9.html", + "ssec-relnotes-2.0": "release-notes/rl-2.0.html", + "ssec-relnotes-2.1": "release-notes/rl-2.1.html", + "ssec-relnotes-2.2": "release-notes/rl-2.2.html", + "ssec-relnotes-2.3": "release-notes/rl-2.3.html" + }, + "language/values.html": { + "simple-values": "#primitives", + "lists": "#list", + "strings": "#string", + "lists": "#list", + "attribute-sets": "#attribute-set" + }, + "installation/installing-binary.html": { + "uninstalling": "uninstall.html" + } +}; + +// the following code matches the current page's URL against the set of redirects. +// +// it is written to minimize the latency between page load and redirect. +// therefore we avoid function calls, copying data, and unnecessary loops. +// IMPORTANT: we use stateful array operations and their order matters! +// +// matching URLs is more involved than it should be: +// +// 1. `document.location.pathname` can have an arbitrary prefix. +// +// 2. `path_to_root` is set by mdBook. it consists only of `../`s and +// determines the depth of `` relative to the prefix: +// +// `document.location.pathname` +// |------------------------------| +// ///[[.html]][#] +// |----| +// `path_to_root` has same number of path segments +// +// source: https://phaiax.github.io/mdBook/format/theme/index-hbs.html#data +// +// 3. the following paths are equivalent: +// +// /foo/bar/ +// /foo/bar/index.html +// /foo/bar/index +// +// 4. the following paths are also equivalent: +// +// /foo/bar/baz +// /foo/bar/baz.html +// + +let segments = document.location.pathname.split('/'); + +let file = segments.pop(); + +// normalize file name +if (file === '') { file = "index.html"; } +else if (!file.endsWith('.html')) { file = file + '.html'; } + +segments.push(file); + +// use `path_to_root` to discern prefix from path. +const depth = path_to_root.split('/').length; + +// remove segments containing prefix. the following works because +// 1. the original `document.location.pathname` is absolute, +// hence first element of `segments` is always empty. +// 2. last element of splitting `path_to_root` is also always empty. +// 3. last element of `segments` is the file name. +// +// visual example: +// +// '/foo/bar/baz.html'.split('/') -> [ '', 'foo', 'bar', 'baz.html' ] +// '../'.split('/') -> [ '..', '' ] +// +// the following operations will then result in +// +// path = 'bar/baz.html' +// +segments.splice(0, segments.length - depth); +const path = segments.join('/'); + +// anchor starts with the hash character (`#`), +// but our redirect declarations don't, so we strip it. +// example: +// document.location.hash -> '#foo' +// document.location.hash.substring(1) -> 'foo' +const anchor = document.location.hash.substring(1); + +const redirect = redirects[path]; +if (redirect) { + const target = redirect[anchor]; + if (target) { + document.location.href = target; + } +} diff -Nru nix-2.8.0/doc/manual/src/advanced-topics/diff-hook.md nix-2.16.1+dfsg/doc/manual/src/advanced-topics/diff-hook.md --- nix-2.8.0/doc/manual/src/advanced-topics/diff-hook.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/advanced-topics/diff-hook.md 2023-06-06 10:55:23.000000000 +0000 @@ -48,13 +48,13 @@ code of 0: ```console -$ nix-build ./deterministic.nix -A stable +$ nix-build ./deterministic.nix --attr stable this derivation will be built: /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable -$ nix-build ./deterministic.nix -A stable --check +$ nix-build ./deterministic.nix --attr stable --check checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable ``` @@ -63,13 +63,13 @@ 1: ```console -$ nix-build ./deterministic.nix -A unstable +$ nix-build ./deterministic.nix --attr unstable this derivation will be built: /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable -$ nix-build ./deterministic.nix -A unstable --check +$ nix-build ./deterministic.nix --attr unstable --check checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs @@ -89,7 +89,7 @@ build's output in a special, `.check` path: ```console -$ nix-build ./deterministic.nix -A unstable --check --keep-failed +$ nix-build ./deterministic.nix --attr unstable --check --keep-failed checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... note: keeping build directory '/tmp/nix-build-unstable.drv-0' error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may @@ -101,7 +101,7 @@ `/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check` output. Nix has copied the build results to that directory where you can examine it. -> **Note** +> []{#check-dirs-are-unregistered} **Note** > > Check paths are not protected against garbage collection, and this > path will be deleted on the next garbage collection. @@ -121,37 +121,3 @@ are not valid, so checking is not possible Run the build without `--check`, and then try with `--check` again. - -# Automatic and Optionally Enforced Determinism Verification - -Automatically verify every build at build time by executing the build -multiple times. - -Setting `repeat` and `enforce-determinism` in your `nix.conf` permits -the automated verification of every build Nix performs. - -The following configuration will run each build three times, and will -require the build to be deterministic: - - enforce-determinism = true - repeat = 2 - -Setting `enforce-determinism` to false as in the following -configuration will run the build multiple times, execute the build -hook, but will allow the build to succeed even if it does not build -reproducibly: - - enforce-determinism = false - repeat = 1 - -An example output of this configuration: - -```console -$ nix-build ./test.nix -A unstable -this derivation will be built: - /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv -building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)... -building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)... -output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round -/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable -``` diff -Nru nix-2.8.0/doc/manual/src/advanced-topics/distributed-builds.md nix-2.16.1+dfsg/doc/manual/src/advanced-topics/distributed-builds.md --- nix-2.8.0/doc/manual/src/advanced-topics/distributed-builds.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/advanced-topics/distributed-builds.md 2023-06-06 10:55:23.000000000 +0000 @@ -12,14 +12,14 @@ test whether connecting to the remote Nix instance works, e.g. ```console -$ nix ping-store --store ssh://mac +$ nix store ping --store ssh://mac ``` will try to connect to the machine named `mac`. It is possible to specify an SSH identity file as part of the remote store URI, e.g. ```console -$ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key +$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key ``` Since builds should be non-interactive, the key should not have a @@ -38,11 +38,9 @@ > **Warning** > -> If you are building via the Nix daemon, it is the Nix daemon user -> account (that is, `root`) that should have SSH access to the remote -> machine. If you can’t or don’t want to configure `root` to be able to -> access to remote machine, you can use a private Nix store instead by -> passing e.g. `--store ~/my-nix`. +> If you are building via the Nix daemon, it is the Nix daemon user account (that is, `root`) that should have SSH access to a user (not necessarily `root`) on the remote machine. +> +> If you can’t or don’t want to configure `root` to be able to access the remote machine, you can use a private Nix store instead by passing e.g. `--store ~/my-nix` when running a Nix command from the local machine. The list of remote machines can be specified on the command line or in the Nix configuration file. The former is convenient for testing. For diff -Nru nix-2.8.0/doc/manual/src/advanced-topics/post-build-hook.md nix-2.16.1+dfsg/doc/manual/src/advanced-topics/post-build-hook.md --- nix-2.8.0/doc/manual/src/advanced-topics/post-build-hook.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/advanced-topics/post-build-hook.md 2023-06-06 10:55:23.000000000 +0000 @@ -33,12 +33,17 @@ example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= ``` -Then, add the public key and the cache URL to your `nix.conf`'s -`trusted-public-keys` and `substituters` options: +Then update [`nix.conf`](../command-ref/conf-file.md) on any machine that will access the cache. +Add the cache URL to [`substituters`](../command-ref/conf-file.md#conf-substituters) and the public key to [`trusted-public-keys`](../command-ref/conf-file.md#conf-trusted-public-keys): substituters = https://cache.nixos.org/ s3://example-nix-cache trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= +Machines that build for the cache must sign derivations using the private key. +On those machines, add the path to the key file to the [`secret-key-files`](../command-ref/conf-file.md#conf-secret-key-files) field in their [`nix.conf`](../command-ref/conf-file.md): + + secret-key-files = /etc/nix/key.private + We will restart the Nix daemon in a later step. # Implementing the build hook @@ -52,14 +57,12 @@ set -f # disable globbing export IFS=' ' -echo "Signing paths" $OUT_PATHS -nix store sign --key-file /etc/nix/key.private $OUT_PATHS echo "Uploading paths" $OUT_PATHS -exec nix copy --to 's3://example-nix-cache' $OUT_PATHS +exec nix copy --to "s3://example-nix-cache" $OUT_PATHS ``` > **Note** -> +> > The `$OUT_PATHS` variable is a space-separated list of Nix store > paths. In this case, we expect and want the shell to perform word > splitting to make each output path its own argument to `nix @@ -87,7 +90,7 @@ Build any derivation, for example: ```console -$ nix-build -E '(import {}).writeText "example" (builtins.toString builtins.currentTime)' +$ nix-build --expr '(import {}).writeText "example" (builtins.toString builtins.currentTime)' this derivation will be built: /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... diff -Nru nix-2.8.0/doc/manual/src/architecture/architecture.md nix-2.16.1+dfsg/doc/manual/src/architecture/architecture.md --- nix-2.8.0/doc/manual/src/architecture/architecture.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/architecture/architecture.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,115 @@ +# Architecture + +This chapter describes how Nix works. +It should help users understand why Nix behaves as it does, and it should help developers understand how to modify Nix and how to write similar tools. + +## Overview + +Nix consists of [hierarchical layers]. + +[hierarchical layers]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers + +The following [concept map] shows its main components (rectangles), the objects they operate on (rounded rectangles), and their interactions (connecting phrases): + +[concept map]: https://en.m.wikipedia.org/wiki/Concept_map + +``` + + .----------------. + | Nix expression |----------. + '----------------' | + | passed to + | | ++----------|-------------------|--------------------------------+ +| Nix | V | +| | +-------------------------+ | +| | | commmand line interface |------. | +| | +-------------------------+ | | +| | | | | +| evaluated by calls manages | +| | | | | +| | V | | +| | +--------------------+ | | +| '-------->| language evaluator | | | +| +--------------------+ | | +| | | | +| produces | | +| | V | +| +----------------------------|------------------------------+ | +| | store | | | +| | referenced by V builds | | +| | .-------------. .------------. .--------------. | | +| | | build input |----->| build plan |----->| build result | | | +| | '-------------' '------------' '--------------' | | +| +-------------------------------------------------|---------+ | ++---------------------------------------------------|-----------+ + | + represented as + | + V + .---------------. + | file | + '---------------' +``` + +At the top is the [command line interface](../command-ref/command-ref.md) that drives the underlying layers. + +The [Nix language](../language/index.md) evaluator transforms Nix expressions into self-contained *build plans*, which are used to derive *build results* from referenced *build inputs*. + +The command line interface and Nix expressions are what users deal with most. + +> **Note** +> The Nix language itself does not have a notion of *packages* or *configurations*. +> As far as we are concerned here, the inputs and results of a build plan are just data. + +Underlying the command line interface and the Nix language evaluator is the [Nix store](../glossary.md#gloss-store), a mechanism to keep track of build plans, data, and references between them. +It can also execute build plans to produce new data, which are made available to the operating system as files. + +A build plan itself is a series of *build tasks*, together with their build inputs. + +> **Important** +> A build task in Nix is called [derivation](../glossary.md#gloss-derivation). + +Each build task has a special build input executed as *build instructions* in order to perform the build. +The result of a build task can be input to another build task. + +The following [data flow diagram] shows a build plan for illustration. +Build inputs used as instructions to a build task are marked accordingly: + +[data flow diagram]: https://en.m.wikipedia.org/wiki/Data-flow_diagram + +``` ++--------------------------------------------------------------------+ +| build plan | +| | +| .-------------. | +| | build input |---------. | +| '-------------' | | +| instructions | +| | | +| v | +| .-------------. .----------. | +| | build input |-->( build task )-------. | +| '-------------' '----------' | | +| instructions | +| | | +| v | +| .-------------. .----------. .--------------. | +| | build input |---------. ( build task )--->| build result | | +| '-------------' | '----------' '--------------' | +| instructions ^ | +| | | | +| v | | +| .-------------. .----------. | | +| | build input |-->( build task )-------' | +| '-------------' '----------' | +| ^ | +| | | +| | | +| .-------------. | | +| | build input |---------' | +| '-------------' | +| | ++--------------------------------------------------------------------+ +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/env-common.md nix-2.16.1+dfsg/doc/manual/src/command-ref/env-common.md --- nix-2.8.0/doc/manual/src/command-ref/env-common.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/env-common.md 2023-06-06 10:55:23.000000000 +0000 @@ -2,49 +2,29 @@ Most Nix commands interpret the following environment variables: - - `IN_NIX_SHELL`\ + - [`IN_NIX_SHELL`](#env-IN_NIX_SHELL)\ Indicator that tells if the current environment was set up by - `nix-shell`. Since Nix 2.0 the values are `"pure"` and `"impure"` + `nix-shell`. It can have the values `pure` or `impure`. - - `NIX_PATH`\ - A colon-separated list of directories used to look up Nix - expressions enclosed in angle brackets (i.e., ``). For - instance, the value + - [`NIX_PATH`](#env-NIX_PATH)\ + A colon-separated list of directories used to look up the location of Nix + expressions using [paths](@docroot@/language/values.md#type-path) + enclosed in angle brackets (i.e., ``), + e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the + [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). - /home/eelco/Dev:/etc/nixos + If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in this order. It is also possible to match paths - against a prefix. For example, the value + 1. `$HOME/.nix-defexpr/channels` + 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` + 3. `/nix/var/nix/profiles/per-user/root/channels` - nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos + If `NIX_PATH` is set to an empty string, resolving search paths will always fail. + For example, attempting to use `` will produce: - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. + error: file 'nixpkgs' was not found in the Nix search path - If a path in the Nix search path starts with `http://` or - `https://`, it is interpreted as the URL of a tarball that will be - downloaded and unpacked to a temporary location. The tarball must - consist of a single top-level directory. For example, setting - `NIX_PATH` to - - nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - - tells Nix to download and use the current contents of the - `master` branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official nixos.org channels (see - [the manual for `nix-channel`](nix-channel.md)) can be abbreviated - as `channel:`. For instance, the following two - values of `NIX_PATH` are equivalent: - - nixpkgs=channel:nixos-21.05 - nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - - The Nix search path can also be extended using the `-I` option to - many Nix commands, which takes precedence over `NIX_PATH`. - - - `NIX_IGNORE_SYMLINK_STORE`\ + - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE)\ Normally, the Nix store directory (typically `/nix/store`) is not allowed to contain any symlink components. This is to prevent “impure” builds. Builders sometimes “canonicalise” paths by @@ -66,59 +46,79 @@ Consult the mount 8 manual page for details. - - `NIX_STORE_DIR`\ + - [`NIX_STORE_DIR`](#env-NIX_STORE_DIR)\ Overrides the location of the Nix store (default `prefix/store`). - - `NIX_DATA_DIR`\ + - [`NIX_DATA_DIR`](#env-NIX_DATA_DIR)\ Overrides the location of the Nix static data directory (default `prefix/share`). - - `NIX_LOG_DIR`\ + - [`NIX_LOG_DIR`](#env-NIX_LOG_DIR)\ Overrides the location of the Nix log directory (default `prefix/var/log/nix`). - - `NIX_STATE_DIR`\ + - [`NIX_STATE_DIR`](#env-NIX_STATE_DIR)\ Overrides the location of the Nix state directory (default `prefix/var/nix`). - - `NIX_CONF_DIR`\ + - [`NIX_CONF_DIR`](#env-NIX_CONF_DIR)\ Overrides the location of the system Nix configuration directory (default `prefix/etc/nix`). - - `NIX_CONFIG`\ + - [`NIX_CONFIG`](#env-NIX_CONFIG)\ Applies settings from Nix configuration from the environment. The content is treated as if it was read from a Nix configuration file. Settings are separated by the newline character. - - `NIX_USER_CONF_FILES`\ - Overrides the location of the user Nix configuration files to load - from (defaults to the XDG spec locations). The variable is treated - as a list separated by the `:` token. + - [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES)\ + Overrides the location of the Nix user configuration files to load from. + + The default are the locations according to the [XDG Base Directory Specification]. + See the [XDG Base Directories](#xdg-base-directories) sub-section for details. + + The variable is treated as a list separated by the `:` token. - - `TMPDIR`\ + - [`TMPDIR`](#env-TMPDIR)\ Use the specified directory to store temporary files. In particular, this includes temporary build directories; these can take up substantial amounts of disk space. The default is `/tmp`. - - `NIX_REMOTE`\ + - [`NIX_REMOTE`](#env-NIX_REMOTE)\ This variable should be set to `daemon` if you want to use the Nix daemon to execute Nix operations. This is necessary in [multi-user - Nix installations](../installation/multi-user.md). If the Nix + Nix installations](@docroot@/installation/multi-user.md). If the Nix daemon's Unix socket is at some non-standard path, this variable should be set to `unix://path/to/socket`. Otherwise, it should be left unset. - - `NIX_SHOW_STATS`\ + - [`NIX_SHOW_STATS`](#env-NIX_SHOW_STATS)\ If set to `1`, Nix will print some evaluation statistics, such as the number of values allocated. - - `NIX_COUNT_CALLS`\ + - [`NIX_COUNT_CALLS`](#env-NIX_COUNT_CALLS)\ If set to `1`, Nix will print how often functions were called during Nix expression evaluation. This is useful for profiling your Nix expressions. - - `GC_INITIAL_HEAP_SIZE`\ + - [`GC_INITIAL_HEAP_SIZE`](#env-GC_INITIAL_HEAP_SIZE)\ If Nix has been configured to use the Boehm garbage collector, this variable sets the initial size of the heap in bytes. It defaults to 384 MiB. Setting it to a low value reduces memory consumption, but will increase runtime due to the overhead of garbage collection. + +## XDG Base Directories + +Nix follows the [XDG Base Directory Specification]. + +For backwards compatibility, Nix commands will follow the standard only when [`use-xdg-base-directories`] is enabled. +[New Nix commands](@docroot@/command-ref/new-cli/nix.md) (experimental) conform to the standard by default. + +The following environment variables are used to determine locations of various state and configuration files: + +- [`XDG_CONFIG_HOME`]{#env-XDG_CONFIG_HOME} (default `~/.config`) +- [`XDG_STATE_HOME`]{#env-XDG_STATE_HOME} (default `~/.local/state`) +- [`XDG_CACHE_HOME`]{#env-XDG_CACHE_HOME} (default `~/.cache`) + + +[XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html +[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories diff -Nru nix-2.8.0/doc/manual/src/command-ref/experimental-commands.md nix-2.16.1+dfsg/doc/manual/src/command-ref/experimental-commands.md --- nix-2.8.0/doc/manual/src/command-ref/experimental-commands.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/experimental-commands.md 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,6 @@ # Experimental Commands -This section lists experimental commands. +This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command). > **Warning** > diff -Nru nix-2.8.0/doc/manual/src/command-ref/files/channels.md nix-2.16.1+dfsg/doc/manual/src/command-ref/files/channels.md --- nix-2.8.0/doc/manual/src/command-ref/files/channels.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/files/channels.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,26 @@ +## Channels + +A directory containing symlinks to Nix channels, managed by [`nix-channel`]: + +- `$XDG_STATE_HOME/nix/profiles/channels` for regular users +- `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root` + +[`nix-channel`] uses a [profile](@docroot@/command-ref/files/profiles.md) to store channels. +This profile contains symlinks to the contents of those channels. + +## Subscribed channels + +The list of subscribed channels is stored in + +- `~/.nix-channels` +- `$XDG_STATE_HOME/nix/channels` if [`use-xdg-base-directories`] is set to `true` + +in the following format: + +``` + +... +``` + +[`nix-channel`]: @docroot@/command-ref/nix-channel.md +[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories diff -Nru nix-2.8.0/doc/manual/src/command-ref/files/default-nix-expression.md nix-2.16.1+dfsg/doc/manual/src/command-ref/files/default-nix-expression.md --- nix-2.8.0/doc/manual/src/command-ref/files/default-nix-expression.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/files/default-nix-expression.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,52 @@ +## Default Nix expression + +The source for the default [Nix expressions](@docroot@/language/index.md) used by [`nix-env`]: + +- `~/.nix-defexpr` +- `$XDG_STATE_HOME/nix/defexpr` if [`use-xdg-base-directories`] is set to `true`. + +It is loaded as follows: + +- If the default expression is a file, it is loaded as a Nix expression. +- If the default expression is a directory containing a `default.nix` file, that `default.nix` file is loaded as a Nix expression. +- If the default expression is a directory without a `default.nix` file, then its contents (both files and subdirectories) are loaded as Nix expressions. + The expressions are combined into a single attribute set, each expression under an attribute with the same name as the original file or subdirectory. + Subdirectories without a `default.nix` file are traversed recursively in search of more Nix expressions, but the names of these intermediate directories are not added to the attribute paths of the default Nix expression. + +Then, the resulting expression is interpreted like this: + +- If the expression is an attribute set, it is used as the default Nix expression. +- If the expression is a function, an empty set is passed as argument and the return value is used as the default Nix expression. + + +For example, if the default expression contains two files, `foo.nix` and `bar.nix`, then the default Nix expression will be equivalent to + +```nix +{ + foo = import ~/.nix-defexpr/foo.nix; + bar = import ~/.nix-defexpr/bar.nix; +} +``` + +The file [`manifest.nix`](@docroot@/command-ref/files/manifest.nix.md) is always ignored. + +The command [`nix-channel`] places a symlink to the user's current [channels profile](@docroot@/command-ref/files/channels.md) in this directory. +This makes all subscribed channels available as attributes in the default expression. + +## User channel link + +A symlink that ensures that [`nix-env`] can find your channels: + +- `~/.nix-defexpr/channels` +- `$XDG_STATE_HOME/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`. + +This symlink points to: + +- `$XDG_STATE_HOME/profiles/channels` for regular users +- `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root` + +In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, which links to the channels of the root user.[`nix-env`]: ../nix-env.md + +[`nix-env`]: @docroot@/command-ref/nix-env.md +[`nix-channel`]: @docroot@/command-ref/nix-channel.md +[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories diff -Nru nix-2.8.0/doc/manual/src/command-ref/files/manifest.json.md nix-2.16.1+dfsg/doc/manual/src/command-ref/files/manifest.json.md --- nix-2.8.0/doc/manual/src/command-ref/files/manifest.json.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/files/manifest.json.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,45 @@ +## `manifest.json` + +The manifest file records the provenance of the packages that are installed in a [profile](./profiles.md) managed by [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) (experimental). + +Here is an example of what the file might look like after installing `zoom-us` from Nixpkgs: + +```json +{ + "version": 1, + "elements": [ + { + "active": true, + "attrPath": "legacyPackages.x86_64-linux.zoom-us", + "originalUrl": "flake:nixpkgs", + "storePaths": [ + "/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927" + ], + "uri": "github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a" + }, + … + ] +} +``` + +Each object in the array `elements` denotes an installed package and +has the following fields: + +* `originalUrl`: The [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md) specified by + the user at the time of installation (e.g. `nixpkgs`). This is also + the flake reference that will be used by `nix profile upgrade`. + +* `uri`: The locked flake reference to which `originalUrl` resolved. + +* `attrPath`: The flake output attribute that provided this + package. Note that this is not necessarily the attribute that the + user specified, but the one resulting from applying the default + attribute paths and prefixes; for instance, `hello` might resolve to + `packages.x86_64-linux.hello` and the empty string to + `packages.x86_64-linux.default`. + +* `storePath`: The paths in the Nix store containing the package. + +* `active`: Whether the profile contains symlinks to the files of this + package. If set to false, the package is kept in the Nix store, but + is not "visible" in the profile's symlink tree. diff -Nru nix-2.8.0/doc/manual/src/command-ref/files/manifest.nix.md nix-2.16.1+dfsg/doc/manual/src/command-ref/files/manifest.nix.md --- nix-2.8.0/doc/manual/src/command-ref/files/manifest.nix.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/files/manifest.nix.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,128 @@ +## `manifest.nix` + +The manifest file records the provenance of the packages that are installed in a [profile](./profiles.md) managed by [`nix-env`](@docroot@/command-ref/nix-env.md). + +Here is an example of how this file might look like after installing `hello` from Nixpkgs: + +```nix +[{ + meta = { + available = true; + broken = false; + changelog = + "https://git.savannah.gnu.org/cgit/hello.git/plain/NEWS?h=v2.12.1"; + description = "A program that produces a familiar, friendly greeting"; + homepage = "https://www.gnu.org/software/hello/manual/"; + insecure = false; + license = { + deprecated = false; + free = true; + fullName = "GNU General Public License v3.0 or later"; + redistributable = true; + shortName = "gpl3Plus"; + spdxId = "GPL-3.0-or-later"; + url = "https://spdx.org/licenses/GPL-3.0-or-later.html"; + }; + longDescription = '' + GNU Hello is a program that prints "Hello, world!" when you run it. + It is fully customizable. + ''; + maintainers = [{ + email = "edolstra+nixpkgs@gmail.com"; + github = "edolstra"; + githubId = 1148549; + name = "Eelco Dolstra"; + }]; + name = "hello-2.12.1"; + outputsToInstall = [ "out" ]; + platforms = [ + "i686-cygwin" + "x86_64-cygwin" + "x86_64-darwin" + "i686-darwin" + "aarch64-darwin" + "armv7a-darwin" + "i686-freebsd13" + "x86_64-freebsd13" + "aarch64-genode" + "i686-genode" + "x86_64-genode" + "x86_64-solaris" + "js-ghcjs" + "aarch64-linux" + "armv5tel-linux" + "armv6l-linux" + "armv7a-linux" + "armv7l-linux" + "i686-linux" + "m68k-linux" + "microblaze-linux" + "microblazeel-linux" + "mipsel-linux" + "mips64el-linux" + "powerpc64-linux" + "powerpc64le-linux" + "riscv32-linux" + "riscv64-linux" + "s390-linux" + "s390x-linux" + "x86_64-linux" + "mmix-mmixware" + "aarch64-netbsd" + "armv6l-netbsd" + "armv7a-netbsd" + "armv7l-netbsd" + "i686-netbsd" + "m68k-netbsd" + "mipsel-netbsd" + "powerpc-netbsd" + "riscv32-netbsd" + "riscv64-netbsd" + "x86_64-netbsd" + "aarch64_be-none" + "aarch64-none" + "arm-none" + "armv6l-none" + "avr-none" + "i686-none" + "microblaze-none" + "microblazeel-none" + "msp430-none" + "or1k-none" + "m68k-none" + "powerpc-none" + "powerpcle-none" + "riscv32-none" + "riscv64-none" + "rx-none" + "s390-none" + "s390x-none" + "vc4-none" + "x86_64-none" + "i686-openbsd" + "x86_64-openbsd" + "x86_64-redox" + "wasm64-wasi" + "wasm32-wasi" + "x86_64-windows" + "i686-windows" + ]; + position = + "/nix/store/7niq32w715567hbph0q13m5lqna64c1s-nixos-unstable.tar.gz/nixos-unstable.tar.gz/pkgs/applications/misc/hello/default.nix:34"; + unfree = false; + unsupported = false; + }; + name = "hello-2.12.1"; + out = { + outPath = "/nix/store/260q5867crm1xjs4khgqpl6vr9kywql1-hello-2.12.1"; + }; + outPath = "/nix/store/260q5867crm1xjs4khgqpl6vr9kywql1-hello-2.12.1"; + outputs = [ "out" ]; + system = "x86_64-linux"; + type = "derivation"; +}] +``` + +Each element in this list corresponds to an installed package. +It incorporates some attributes of the original derivation, including `meta`, `name`, `out`, `outPath`, `outputs`, `system`. +This information is used by Nix for querying and updating the package. diff -Nru nix-2.8.0/doc/manual/src/command-ref/files/profiles.md nix-2.16.1+dfsg/doc/manual/src/command-ref/files/profiles.md --- nix-2.8.0/doc/manual/src/command-ref/files/profiles.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/files/profiles.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,74 @@ +## Profiles + +A directory that contains links to profiles managed by [`nix-env`] and [`nix profile`]: + +- `$XDG_STATE_HOME/nix/profiles` for regular users +- `$NIX_STATE_DIR/profiles/per-user/root` if the user is `root` + +A profile is a directory of symlinks to files in the Nix store. + +### Filesystem layout + +Profiles are versioned as follows. When using a profile named *path*, *path* is a symlink to *path*`-`*N*`-link`, where *N* is the version of the profile. +In turn, *path*`-`*N*`-link` is a symlink to a path in the Nix store. +For example: + +```console +$ ls -l ~alice/.local/state/nix/profiles/profile* +lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile -> profile-7-link +lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /home/alice/.local/state/nix/profiles/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile +lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /home/alice/.local/state/nix/profiles/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile +lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile +``` + +Each of these symlinks is a root for the Nix garbage collector. + +The contents of the store path corresponding to each version of the +profile is a tree of symlinks to the files of the installed packages, +e.g. + +```console +$ ll -R ~eelco/.local/state/nix/profiles/profile-7-link/ +/home/eelco/.local/state/nix/profiles/profile-7-link/: +total 20 +dr-xr-xr-x 2 root root 4096 Jan 1 1970 bin +-r--r--r-- 2 root root 1402 Jan 1 1970 manifest.nix +dr-xr-xr-x 4 root root 4096 Jan 1 1970 share + +/home/eelco/.local/state/nix/profiles/profile-7-link/bin: +total 20 +lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium +lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify +lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us + +/home/eelco/.local/state/nix/profiles/profile-7-link/share/applications: +total 12 +lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop +lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop +lrwxrwxrwx 3 root root 107 Jan 1 1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop + +… +``` + +Each profile version contains a manifest file: +- [`manifest.nix`](@docroot@/command-ref/files/manifest.nix.md) used by [`nix-env`](@docroot@/command-ref/nix-env.md). +- [`manifest.json`](@docroot@/command-ref/files/manifest.json.md) used by [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) (experimental). + +## User profile link + +A symbolic link to the user's current profile: + +- `~/.nix-profile` +- `$XDG_STATE_HOME/nix/profile` if [`use-xdg-base-directories`] is set to `true`. + +By default, this symlink points to: + +- `$XDG_STATE_HOME/nix/profiles/profile` for regular users +- `$NIX_STATE_DIR/profiles/per-user/root/profile` for `root` + +The `PATH` environment variable should include `/bin` subdirectory of the profile link (e.g. `~/.nix-profile/bin`) for the user environment to be visible to the user. +The [installer](@docroot@/installation/installing-binary.md) sets this up by default, unless you enable [`use-xdg-base-directories`]. + +[`nix-env`]: @docroot@/command-ref/nix-env.md +[`nix profile`]: @docroot@/command-ref/new-cli/nix3-profile.md +[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-build.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-build.md --- nix-2.8.0/doc/manual/src/command-ref/nix-build.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-build.md 2023-06-06 10:55:23.000000000 +0000 @@ -12,6 +12,12 @@ [`--dry-run`] [{`--out-link` | `-o`} *outlink*] +# Disambiguation + +This man page describes the command `nix-build`, which is distinct from `nix +build`. For documentation on the latter, run `nix build --help` or see `man +nix3-build`. + # Description The `nix-build` command builds the derivations described by the Nix @@ -31,10 +37,12 @@ `nix-build` is essentially a wrapper around [`nix-instantiate`](nix-instantiate.md) (to translate a high-level Nix -expression to a low-level store derivation) and [`nix-store ---realise`](nix-store.md#operation---realise) (to build the store +expression to a low-level [store derivation]) and [`nix-store +--realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store derivation). +[store derivation]: ../glossary.md#gloss-store-derivation + > **Warning** > > The result of the build is automatically registered as a root of the @@ -43,29 +51,32 @@ # Options -All options not listed here are passed to `nix-store ---realise`, except for `--arg` and `--attr` / `-A` which are passed to -`nix-instantiate`. +All options not listed here are passed to `nix-store --realise`, +except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. + + - [`--no-out-link`](#opt-no-out-link) - - `--no-out-link`\ Do not create a symlink to the output path. Note that as a result the output does not become a root of the garbage collector, and so - might be deleted by `nix-store - --gc`. + might be deleted by `nix-store --gc`. + + - [`--dry-run`](#opt-dry-run) - - `--dry-run`\ Show what store paths would be built or downloaded. - - `--out-link` / `-o` *outlink*\ + - [`--out-link`](#opt-out-link) / `-o` *outlink* + Change the name of the symlink to the output path created from `result` to *outlink*. -The following common options are supported: +{{#include ./opt-common.md}} + +{{#include ./env-common.md}} # Examples ```console -$ nix-build '' -A firefox +$ nix-build '' --attr firefox store derivation is /nix/store/qybprl8sz2lc...-firefox-1.5.0.7.drv /nix/store/d18hyl92g30l...-firefox-1.5.0.7 @@ -80,7 +91,7 @@ (first) output. You can also build all outputs: ```console -$ nix-build '' -A openssl.all +$ nix-build '' --attr openssl.all ``` This will create a symlink for each output named `result-outputname`. @@ -90,7 +101,7 @@ specific output: ```console -$ nix-build '' -A openssl.man +$ nix-build '' --attr openssl.man ``` This will create a symlink `result-man`. @@ -98,7 +109,7 @@ Build a Nix expression given on the command line: ```console -$ nix-build -E 'with import { }; runCommand "foo" { } "echo bar > $out"' +$ nix-build --expr 'with import { }; runCommand "foo" { } "echo bar > $out"' $ cat ./result bar ``` @@ -107,5 +118,5 @@ branch of Nixpkgs: ```console -$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello +$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz --attr hello ``` diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-channel.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-channel.md --- nix-2.8.0/doc/manual/src/command-ref/nix-channel.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-channel.md 2023-06-06 10:55:23.000000000 +0000 @@ -22,6 +22,9 @@ channels. If *name* is omitted, it defaults to the last component of *url*, with the suffixes `-stable` or `-unstable` removed. + A channel URL must point to a directory containing a file `nixexprs.tar.gz`. + At the top level, that tarball must contain a single directory with a `default.nix` file that serves as the channel’s entry point. + - `--remove` *name*\ Removes the channel named *name* from the list of subscribed channels. @@ -45,6 +48,16 @@ The list of subscribed channels is stored in `~/.nix-channels`. +{{#include ./opt-common.md}} + +{{#include ./env-common.md}} + +# Files + +`nix-channel` operates on the following files. + +{{#include ./files/channels.md}} + # Examples To subscribe to the Nixpkgs channel and install the GNU Hello package: @@ -52,45 +65,18 @@ ```console $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable $ nix-channel --update -$ nix-env -iA nixpkgs.hello +$ nix-env --install --attr nixpkgs.hello ``` You can revert channel updates using `--rollback`: ```console -$ nix-instantiate --eval -E '(import {}).lib.version' +$ nix-instantiate --eval --expr '(import {}).lib.version' "14.04.527.0e935f1" $ nix-channel --rollback switching from generation 483 to 482 -$ nix-instantiate --eval -E '(import {}).lib.version' +$ nix-instantiate --eval --expr '(import {}).lib.version' "14.04.526.dbadfad" ``` - -# Files - - - `/nix/var/nix/profiles/per-user/username/channels`\ - `nix-channel` uses a `nix-env` profile to keep track of previous - versions of the subscribed channels. Every time you run `nix-channel - --update`, a new channel generation (that is, a symlink to the - channel Nix expressions in the Nix store) is created. This enables - `nix-channel --rollback` to revert to previous versions. - - - `~/.nix-defexpr/channels`\ - This is a symlink to - `/nix/var/nix/profiles/per-user/username/channels`. It ensures that - `nix-env` can find your channels. In a multi-user installation, you - may also have `~/.nix-defexpr/channels_root`, which links to the - channels of the root user. - -# Channel format - -A channel URL should point to a directory containing the following -files: - - - `nixexprs.tar.xz`\ - A tarball containing Nix expressions and files referenced by them - (such as build scripts and patches). At the top level, the tarball - should contain a single directory. That directory must contain a - file `default.nix` that serves as the channel’s “entry point”. diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-collect-garbage.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-collect-garbage.md --- nix-2.8.0/doc/manual/src/command-ref/nix-collect-garbage.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-collect-garbage.md 2023-06-06 10:55:23.000000000 +0000 @@ -9,7 +9,7 @@ # Description The command `nix-collect-garbage` is mostly an alias of [`nix-store ---gc`](nix-store.md#operation---gc), that is, it deletes all +--gc`](@docroot@/command-ref/nix-store/gc.md), that is, it deletes all unreachable paths in the Nix store to clean up your system. However, it provides two additional options: `-d` (`--delete-old`), which deletes all old generations of all profiles in `/nix/var/nix/profiles` @@ -20,6 +20,10 @@ of days in all profiles in `/nix/var/nix/profiles` (except for the generations that were active at that point in time). +{{#include ./opt-common.md}} + +{{#include ./env-common.md}} + # Example To delete from the Nix store everything that is not used by the current diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-copy-closure.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-copy-closure.md --- nix-2.8.0/doc/manual/src/command-ref/nix-copy-closure.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-copy-closure.md 2023-06-06 10:55:23.000000000 +0000 @@ -30,8 +30,8 @@ appropriate password or passphrase. In fact, you may be asked _twice_ because `nix-copy-closure` currently connects twice to the remote machine, first to get the set of paths missing on the target machine, -and second to send the dump of those paths. If this bothers you, use -`ssh-agent`. +and second to send the dump of those paths. When using public key +authentication, you can avoid typing the passphrase with `ssh-agent`. # Options @@ -47,7 +47,9 @@ Enable compression of the SSH connection. - `--include-outputs`\ - Also copy the outputs of store derivations included in the closure. + Also copy the outputs of [store derivation]s included in the closure. + + [store derivation]: ../glossary.md#gloss-store-derivation - `--use-substitutes` / `-s`\ Attempt to download missing paths on the target machine using Nix’s @@ -61,12 +63,16 @@ - `-v`\ Show verbose output. +{{#include ./opt-common.md}} + # Environment variables - `NIX_SSHOPTS`\ Additional options to be passed to `ssh` on the command line. +{{#include ./env-common.md}} + # Examples Copy Firefox with all its dependencies to a remote machine: @@ -81,5 +87,5 @@ ```console $ nix-copy-closure --from alice@itchy.labs \ /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 -$ nix-env -i /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 +$ nix-env --install /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 ``` diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-daemon.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-daemon.md --- nix-2.8.0/doc/manual/src/command-ref/nix-daemon.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-daemon.md 2023-06-06 10:55:23.000000000 +0000 @@ -8,6 +8,6 @@ # Description -The Nix daemon is necessary in multi-user Nix installations. It performs -build actions and other operations on the Nix store on behalf of +The Nix daemon is necessary in multi-user Nix installations. It runs +build tasks and other operations on the Nix store on behalf of unprivileged users. diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env/delete-generations.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/delete-generations.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env/delete-generations.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/delete-generations.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,46 @@ +# Name + +`nix-env --delete-generations` - delete profile generations + +# Synopsis + +`nix-env` `--delete-generations` *generations* + +# Description + +This operation deletes the specified generations of the current profile. +The generations can be a list of generation numbers, the special value +`old` to delete all non-current generations, a value such as `30d` to +delete all generations older than the specified number of days (except +for the generation that was active at that point in time), or a value +such as `+5` to keep the last `5` generations ignoring any newer than +current, e.g., if `30` is the current generation `+5` will delete +generation `25` and all older generations. Periodically deleting old +generations is important to make garbage collection effective. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --delete-generations 3 4 8 +``` + +```console +$ nix-env --delete-generations +5 +``` + +```console +$ nix-env --delete-generations 30d +``` + +```console +$ nix-env --profile other_profile --delete-generations old +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env/env-common.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/env-common.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env/env-common.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/env-common.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,6 @@ +# Environment variables + +- `NIX_PROFILE`\ + Location of the Nix profile. Defaults to the target of the symlink + `~/.nix-profile`, if it exists, or `/nix/var/nix/profiles/default` + otherwise. diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env/install.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/install.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env/install.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/install.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,187 @@ +# Name + +`nix-env --install` - add packages to user environment + +# Synopsis + +`nix-env` {`--install` | `-i`} *args…* + [{`--prebuilt-only` | `-b`}] + [{`--attr` | `-A`}] + [`--from-expression`] [`-E`] + [`--from-profile` *path*] + [`--preserve-installed` | `-P`] + [`--remove-all` | `-r`] + +# Description + +The install operation creates a new user environment, based on the +current generation of the active profile, to which a set of store paths +described by *args* is added. The arguments *args* map to store paths in +a number of possible ways: + + - By default, *args* is a set of derivation names denoting derivations + in the active Nix expression. These are realised, and the resulting + output paths are installed. Currently installed derivations with a + name equal to the name of a derivation being added are removed + unless the option `--preserve-installed` is specified. + + If there are multiple derivations matching a name in *args* that + have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the + derivation with the highest *priority* is used. A derivation can + define a priority by declaring the `meta.priority` attribute. This + attribute should be a number, with a higher value denoting a lower + priority. The default priority is `0`. + + If there are multiple matching derivations with the same priority, + then the derivation with the highest version will be installed. + + You can force the installation of multiple derivations with the same + name by being specific about the versions. For instance, `nix-env --install + gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will + probably cause a user environment conflict\!). + + - If `--attr` (`-A`) is specified, the arguments are *attribute + paths* that select attributes from the top-level Nix + expression. This is faster than using derivation names and + unambiguous. To find out the attribute paths of available + packages, use `nix-env --query --available --attr-path `. + + - If `--from-profile` *path* is given, *args* is a set of names + denoting installed store paths in the profile *path*. This is an + easy way to copy user environment elements from one profile to + another. + + - If `--from-expression` is given, *args* are Nix + [functions](@docroot@/language/constructs.md#functions) + that are called with the active Nix expression as their single + argument. The derivations returned by those function calls are + installed. This allows derivations to be specified in an + unambiguous way, which is necessary if there are multiple + derivations with the same name. + + - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are + [realised](@docroot@/command-ref/nix-store/realise.md), and the resulting output paths + are installed. + + - If *args* are store paths that are not store derivations, then these + are [realised](@docroot@/command-ref/nix-store/realise.md) and installed. + + - By default all outputs are installed for each derivation. That can + be reduced by setting `meta.outputsToInstall`. + +# Flags + + - `--prebuilt-only` / `-b`\ + Use only derivations for which a substitute is registered, i.e., + there is a pre-built binary available that can be downloaded in lieu + of building the derivation. Thus, no packages will be built from + source. + + - `--preserve-installed` / `-P`\ + Do not remove derivations with a name matching one of the + derivations being installed. Usually, trying to have two versions of + the same package installed in the same generation of a profile will + lead to an error in building the generation, due to file name + clashes between the two versions. However, this is not the case for + all packages. + + - `--remove-all` / `-r`\ + Remove all previously installed packages first. This is equivalent + to running `nix-env --uninstall '.*'` first, except that everything happens + in a single transaction. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To install a package using a specific attribute path from the active Nix expression: + +```console +$ nix-env --install --attr gcc40mips +installing `gcc-4.0.2' +$ nix-env --install --attr xorg.xorgserver +installing `xorg-server-1.2.0' +``` + +To install a specific version of `gcc` using the derivation name: + +```console +$ nix-env --install gcc-3.3.2 +installing `gcc-3.3.2' +uninstalling `gcc-3.1' +``` + +Using attribute path for selecting a package is preferred, +as it is much faster and there will not be multiple matches. + +Note the previously installed version is removed, since +`--preserve-installed` was not specified. + +To install an arbitrary version: + +```console +$ nix-env --install gcc +installing `gcc-3.3.2' +``` + +To install all derivations in the Nix expression `foo.nix`: + +```console +$ nix-env --file ~/foo.nix --install '.*' +``` + +To copy the store path with symbolic name `gcc` from another profile: + +```console +$ nix-env --install --from-profile /nix/var/nix/profiles/foo gcc +``` + +To install a specific [store derivation] (typically created by +`nix-instantiate`): + +```console +$ nix-env --install /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv +``` + +To install a specific output path: + +```console +$ nix-env --install /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3 +``` + +To install from a Nix expression specified on the command-line: + +```console +$ nix-env --file ./foo.nix --install --expr \ + 'f: (f {system = "i686-linux";}).subversionWithJava' +``` + +I.e., this evaluates to `(f: (f {system = +"i686-linux";}).subversionWithJava) (import ./foo.nix)`, thus selecting +the `subversionWithJava` attribute from the set returned by calling the +function defined in `./foo.nix`. + +A dry-run tells you which paths will be downloaded or built from source: + +```console +$ nix-env --file '' --install --attr hello --dry-run +(dry run; not doing anything) +installing ‘hello-2.10’ +this path will be fetched (0.04 MiB download, 0.19 MiB unpacked): + /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10 + ... +``` + +To install Firefox from the latest revision in the Nixpkgs/NixOS 14.12 +channel: + +```console +$ nix-env --file https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz --install --attr firefox +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env/list-generations.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/list-generations.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env/list-generations.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/list-generations.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,33 @@ +# Name + +`nix-env --list-generations` - list profile generations + +# Synopsis + +`nix-env` `--list-generations` + +# Description + +This operation print a list of all the currently existing generations +for the active profile. These may be switched to using the +`--switch-generation` operation. It also prints the creation date of the +generation, and indicates the current generation. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --list-generations + 95 2004-02-06 11:48:24 + 96 2004-02-06 11:49:01 + 97 2004-02-06 16:22:45 + 98 2004-02-06 16:24:33 (current) +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env/opt-common.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/opt-common.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env/opt-common.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/opt-common.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,35 @@ +# Options + +The following options are allowed for all `nix-env` operations, but may not always have an effect. + + - `--file` / `-f` *path*\ + Specifies the Nix expression (designated below as the *active Nix + expression*) used by the `--install`, `--upgrade`, and `--query + --available` operations to obtain derivations. The default is + `~/.nix-defexpr`. + + If the argument starts with `http://` or `https://`, it is + interpreted as the URL of a tarball that will be downloaded and + unpacked to a temporary location. The tarball must include a single + top-level directory containing at least a file named `default.nix`. + + - `--profile` / `-p` *path*\ + Specifies the profile to be used by those operations that operate on + a profile (designated below as the *active profile*). A profile is a + sequence of user environments called *generations*, one of which is + the *current generation*. + + - `--dry-run`\ + For the `--install`, `--upgrade`, `--uninstall`, + `--switch-generation`, `--delete-generations` and `--rollback` + operations, this flag will cause `nix-env` to print what *would* be + done if this flag had not been specified, without actually doing it. + + `--dry-run` also prints out which paths will be + [substituted](@docroot@/glossary.md) (i.e., downloaded) and which paths + will be built from source (because no substitute is available). + + - `--system-filter` *system*\ + By default, operations such as `--query + --available` show derivations matching any platform. This option + allows you to use derivations for the specified platform *system*. diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env/query.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/query.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env/query.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/query.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,215 @@ +# Name + +`nix-env --query` - display information about packages + +# Synopsis + +`nix-env` {`--query` | `-q`} *names…* + [`--installed` | `--available` | `-a`] + [{`--status` | `-s`}] + [{`--attr-path` | `-P`}] + [`--no-name`] + [{`--compare-versions` | `-c`}] + [`--system`] + [`--drv-path`] + [`--out-path`] + [`--description`] + [`--meta`] + [`--xml`] + [`--json`] + [{`--prebuilt-only` | `-b`}] + [{`--attr` | `-A`} *attribute-path*] + +# Description + +The query operation displays information about either the store paths +that are installed in the current generation of the active profile +(`--installed`), or the derivations that are available for installation +in the active Nix expression (`--available`). It only prints information +about derivations whose symbolic name matches one of *names*. + +The derivations are sorted by their `name` attributes. + +# Source selection + +The following flags specify the set of things on which the query +operates. + + - `--installed`\ + The query operates on the store paths that are installed in the + current generation of the active profile. This is the default. + + - `--available`; `-a`\ + The query operates on the derivations that are available in the + active Nix expression. + +# Queries + +The following flags specify what information to display about the +selected derivations. Multiple flags may be specified, in which case the +information is shown in the order given here. Note that the name of the +derivation is shown unless `--no-name` is specified. + + - `--xml`\ + Print the result in an XML representation suitable for automatic + processing by other tools. The root element is called `items`, which + contains a `item` element for each available or installed + derivation. The fields discussed below are all stored in attributes + of the `item` elements. + + - `--json`\ + Print the result in a JSON representation suitable for automatic + processing by other tools. + + - `--prebuilt-only` / `-b`\ + Show only derivations for which a substitute is registered, i.e., + there is a pre-built binary available that can be downloaded in lieu + of building the derivation. Thus, this shows all packages that + probably can be installed quickly. + + - `--status`; `-s`\ + Print the *status* of the derivation. The status consists of three + characters. The first is `I` or `-`, indicating whether the + derivation is currently installed in the current generation of the + active profile. This is by definition the case for `--installed`, + but not for `--available`. The second is `P` or `-`, indicating + whether the derivation is present on the system. This indicates + whether installation of an available derivation will require the + derivation to be built. The third is `S` or `-`, indicating whether + a substitute is available for the derivation. + + - `--attr-path`; `-P`\ + Print the *attribute path* of the derivation, which can be used to + unambiguously select it using the `--attr` option available in + commands that install derivations like `nix-env --install`. This + option only works together with `--available` + + - `--no-name`\ + Suppress printing of the `name` attribute of each derivation. + + - `--compare-versions` / `-c`\ + Compare installed versions to available versions, or vice versa (if + `--available` is given). This is useful for quickly seeing whether + upgrades for installed packages are available in a Nix expression. A + column is added with the following meaning: + + - `<` *version*\ + A newer version of the package is available or installed. + + - `=` *version*\ + At most the same version of the package is available or + installed. + + - `>` *version*\ + Only older versions of the package are available or installed. + + - `- ?`\ + No version of the package is available or installed. + + - `--system`\ + Print the `system` attribute of the derivation. + + - `--drv-path`\ + Print the path of the [store derivation](@docroot@/glossary.md#gloss-store-derivation). + + - `--out-path`\ + Print the output path of the derivation. + + - `--description`\ + Print a short (one-line) description of the derivation, if + available. The description is taken from the `meta.description` + attribute of the derivation. + + - `--meta`\ + Print all of the meta-attributes of the derivation. This option is + only available with `--xml` or `--json`. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To show installed packages: + +```console +$ nix-env --query +bison-1.875c +docbook-xml-4.2 +firefox-1.0.4 +MPlayer-1.0pre7 +ORBit2-2.8.3 +… +``` + +To show available packages: + +```console +$ nix-env --query --available +firefox-1.0.7 +GConf-2.4.0.1 +MPlayer-1.0pre7 +ORBit2-2.8.3 +… +``` + +To show the status of available packages: + +```console +$ nix-env --query --available --status +-P- firefox-1.0.7 (not installed but present) +--S GConf-2.4.0.1 (not present, but there is a substitute for fast installation) +--S MPlayer-1.0pre3 (i.e., this is not the installed MPlayer, even though the version is the same!) +IP- ORBit2-2.8.3 (installed and by definition present) +… +``` + +To show available packages in the Nix expression `foo.nix`: + +```console +$ nix-env --file ./foo.nix --query --available +foo-1.2.3 +``` + +To compare installed versions to what’s available: + +```console +$ nix-env --query --compare-versions +... +acrobat-reader-7.0 - ? (package is not available at all) +autoconf-2.59 = 2.59 (same version) +firefox-1.0.4 < 1.0.7 (a more recent version is available) +... +``` + +To show all packages with “`zip`” in the name: + +```console +$ nix-env --query --available '.*zip.*' +bzip2-1.0.6 +gzip-1.6 +zip-3.0 +… +``` + +To show all packages with “`firefox`” or “`chromium`” in the name: + +```console +$ nix-env --query --available '.*(firefox|chromium).*' +chromium-37.0.2062.94 +chromium-beta-38.0.2125.24 +firefox-32.0.3 +firefox-with-plugins-13.0.1 +… +``` + +To show all packages in the latest revision of the Nixpkgs repository: + +```console +$ nix-env --file https://github.com/NixOS/nixpkgs/archive/master.tar.gz --query --available +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env/rollback.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/rollback.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env/rollback.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/rollback.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,34 @@ +# Name + +`nix-env --rollback` - set user environment to previous generation + +# Synopsis + +`nix-env` `--rollback` + +# Description + +This operation switches to the “previous” generation of the active +profile, that is, the highest numbered generation lower than the current +generation, if it exists. It is just a convenience wrapper around +`--list-generations` and `--switch-generation`. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --rollback +switching from generation 92 to 91 +``` + +```console +$ nix-env --rollback +error: no generation older than the current (91) exists +``` diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env/set-flag.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/set-flag.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env/set-flag.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/set-flag.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,82 @@ +# Name + +`nix-env --set-flag` - modify meta attributes of installed packages + +# Synopsis + +`nix-env` `--set-flag` *name* *value* *drvnames* + +# Description + +The `--set-flag` operation allows meta attributes of installed packages +to be modified. There are several attributes that can be usefully +modified, because they affect the behaviour of `nix-env` or the user +environment build script: + + - `priority` can be changed to resolve filename clashes. The user + environment build script uses the `meta.priority` attribute of + derivations to resolve filename collisions between packages. Lower + priority values denote a higher priority. For instance, the GCC + wrapper package and the Binutils package in Nixpkgs both have a file + `bin/ld`, so previously if you tried to install both you would get a + collision. Now, on the other hand, the GCC wrapper declares a higher + priority than Binutils, so the former’s `bin/ld` is symlinked in the + user environment. + + - `keep` can be set to `true` to prevent the package from being + upgraded or replaced. This is useful if you want to hang on to an + older version of a package. + + - `active` can be set to `false` to “disable” the package. That is, no + symlinks will be generated to the files of the package, but it + remains part of the profile (so it won’t be garbage-collected). It + can be set back to `true` to re-enable the package. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To prevent the currently installed Firefox from being upgraded: + +```console +$ nix-env --set-flag keep true firefox +``` + +After this, `nix-env --upgrade ` will ignore Firefox. + +To disable the currently installed Firefox, then install a new Firefox +while the old remains part of the profile: + +```console +$ nix-env --query +firefox-2.0.0.9 (the current one) + +$ nix-env --preserve-installed --install firefox-2.0.0.11 +installing `firefox-2.0.0.11' +building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment' +collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox' + and `/nix/store/...-firefox-2.0.0.9/bin/firefox'. +(i.e., can’t have two active at the same time) + +$ nix-env --set-flag active false firefox +setting flag on `firefox-2.0.0.9' + +$ nix-env --preserve-installed --install firefox-2.0.0.11 +installing `firefox-2.0.0.11' + +$ nix-env --query +firefox-2.0.0.11 (the enabled one) +firefox-2.0.0.9 (the disabled one) +``` + +To make files from `binutils` take precedence over files from `gcc`: + +```console +$ nix-env --set-flag priority 5 binutils +$ nix-env --set-flag priority 10 gcc +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env/set.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/set.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env/set.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/set.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,30 @@ +# Name + +`nix-env --set` - set profile to contain a specified derivation + +## Synopsis + +`nix-env` `--set` *drvname* + +## Description + +The `--set` operation modifies the current generation of a profile so +that it contains exactly the specified derivation, and nothing else. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +## Examples + +The following updates a profile such that its current generation will +contain just Firefox: + +```console +$ nix-env --profile /nix/var/nix/profiles/browser --set firefox +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env/switch-generation.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/switch-generation.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env/switch-generation.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/switch-generation.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,33 @@ +# Name + +`nix-env --switch-generation` - set user environment to given profile generation + +# Synopsis + +`nix-env` {`--switch-generation` | `-G`} *generation* + +# Description + +This operation makes generation number *generation* the current +generation of the active profile. That is, if the `profile` is the path +to the active profile, then the symlink `profile` is made to point to +`profile-generation-link`, which is in turn a symlink to the actual user +environment in the Nix store. + +Switching will fail if the specified generation does not exist. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --switch-generation 42 +switching from generation 50 to 42 +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env/switch-profile.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/switch-profile.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env/switch-profile.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/switch-profile.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,26 @@ +# Name + +`nix-env --switch-profile` - set user environment to given profile + +# Synopsis + +`nix-env` {`--switch-profile` | `-S`} *path* + +# Description + +This operation makes *path* the current profile for the user. That is, +the symlink `~/.nix-profile` is made to point to *path*. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --switch-profile ~/my-profile +``` diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env/uninstall.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/uninstall.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env/uninstall.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/uninstall.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,28 @@ +# Name + +`nix-env --uninstall` - remove packages from user environment + +# Synopsis + +`nix-env` {`--uninstall` | `-e`} *drvnames…* + +# Description + +The uninstall operation creates a new user environment, based on the +current generation of the active profile, from which the store paths +designated by the symbolic names *drvnames* are removed. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --uninstall gcc +$ nix-env --uninstall '.*' (remove everything) +``` diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env/upgrade.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/upgrade.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env/upgrade.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env/upgrade.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,141 @@ +# Name + +`nix-env --upgrade` - upgrade packages in user environment + +# Synopsis + +`nix-env` {`--upgrade` | `-u`} *args* + [`--lt` | `--leq` | `--eq` | `--always`] + [{`--prebuilt-only` | `-b`}] + [{`--attr` | `-A`}] + [`--from-expression`] [`-E`] + [`--from-profile` *path*] + [`--preserve-installed` | `-P`] + +# Description + +The upgrade operation creates a new user environment, based on the +current generation of the active profile, in which all store paths are +replaced for which there are newer versions in the set of paths +described by *args*. Paths for which there are no newer versions are +left untouched; this is not an error. It is also not an error if an +element of *args* matches no installed derivations. + +For a description of how *args* is mapped to a set of store paths, see +[`--install`](#operation---install). If *args* describes multiple +store paths with the same symbolic name, only the one with the highest +version is installed. + +# Flags + + - `--lt`\ + Only upgrade a derivation to newer versions. This is the default. + + - `--leq`\ + In addition to upgrading to newer versions, also “upgrade” to + derivations that have the same version. Version are not a unique + identification of a derivation, so there may be many derivations + that have the same version. This flag may be useful to force + “synchronisation” between the installed and available derivations. + + - `--eq`\ + *Only* “upgrade” to derivations that have the same version. This may + not seem very useful, but it actually is, e.g., when there is a new + release of Nixpkgs and you want to replace installed applications + with the same versions built against newer dependencies (to reduce + the number of dependencies floating around on your system). + + - `--always`\ + In addition to upgrading to newer versions, also “upgrade” to + derivations that have the same or a lower version. I.e., derivations + may actually be downgraded depending on what is available in the + active Nix expression. + + - `--prebuilt-only` / `-b`\ + Use only derivations for which a substitute is registered, i.e., + there is a pre-built binary available that can be downloaded in lieu + of building the derivation. Thus, no packages will be built from + source. + + - `--preserve-installed` / `-P`\ + Do not remove derivations with a name matching one of the + derivations being installed. Usually, trying to have two versions of + the same package installed in the same generation of a profile will + lead to an error in building the generation, due to file name + clashes between the two versions. However, this is not the case for + all packages. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --upgrade --attr nixpkgs.gcc +upgrading `gcc-3.3.1' to `gcc-3.4' +``` + +When there are no updates available, nothing will happen: + +```console +$ nix-env --upgrade --attr nixpkgs.pan +``` + +Using `-A` is preferred when possible, as it is faster and unambiguous but +it is also possible to upgrade to a specific version by matching the derivation name: + +```console +$ nix-env --upgrade gcc-3.3.2 --always +upgrading `gcc-3.4' to `gcc-3.3.2' +``` + +To try to upgrade everything +(matching packages based on the part of the derivation name without version): + +```console +$ nix-env --upgrade +upgrading `hello-2.1.2' to `hello-2.1.3' +upgrading `mozilla-1.2' to `mozilla-1.4' +``` + +# Versions + +The upgrade operation determines whether a derivation `y` is an upgrade +of a derivation `x` by looking at their respective `name` attributes. +The names (e.g., `gcc-3.3.1` are split into two parts: the package name +(`gcc`), and the version (`3.3.1`). The version part starts after the +first dash not followed by a letter. `y` is considered an upgrade of `x` +if their package names match, and the version of `y` is higher than that +of `x`. + +The versions are compared by splitting them into contiguous components +of numbers and letters. E.g., `3.3.1pre5` is split into `[3, 3, 1, +"pre", 5]`. These lists are then compared lexicographically (from left +to right). Corresponding components `a` and `b` are compared as follows. +If they are both numbers, integer comparison is used. If `a` is an empty +string and `b` is a number, `a` is considered less than `b`. The special +string component `pre` (for *pre-release*) is considered to be less than +other components. String components are considered less than number +components. Otherwise, they are compared lexicographically (i.e., using +case-sensitive string comparison). + +This is illustrated by the following examples: + + 1.0 < 2.3 + 2.1 < 2.3 + 2.3 = 2.3 + 2.5 > 2.3 + 3.1 > 2.3 + 2.3.1 > 2.3 + 2.3.1 > 2.3a + 2.3pre1 < 2.3 + 2.3pre3 < 2.3pre12 + 2.3a < 2.3c + 2.3pre1 < 2.3c + 2.3pre1 < 2.3q + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-env.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env.md --- nix-2.8.0/doc/manual/src/command-ref/nix-env.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-env.md 2023-06-06 10:55:23.000000000 +0000 @@ -4,15 +4,14 @@ # Synopsis -`nix-env` +`nix-env` *operation* [*options*] [*arguments…*] [`--option` *name* *value*] [`--arg` *name* *value*] [`--argstr` *name* *value*] [{`--file` | `-f`} *path*] - [{`--profile` | `-p`} *path(] + [{`--profile` | `-p`} *path*] [`--system-filter` *system*] [`--dry-run`] - *operation* [*options…*] [*arguments…*] # Description @@ -24,14 +23,36 @@ individual users can switch between different environments. `nix-env` takes exactly one *operation* flag which indicates the -subcommand to be performed. These are documented below. +subcommand to be performed. The following operations are available: + +- [`--install`](./nix-env/install.md) +- [`--upgrade`](./nix-env/upgrade.md) +- [`--uninstall`](./nix-env/uninstall.md) +- [`--set`](./nix-env/set.md) +- [`--set-flag`](./nix-env/set-flag.md) +- [`--query`](./nix-env/query.md) +- [`--switch-profile`](./nix-env/switch-profile.md) +- [`--list-generations`](./nix-env/list-generations.md) +- [`--delete-generations`](./nix-env/delete-generations.md) +- [`--switch-generation`](./nix-env/switch-generation.md) +- [`--rollback`](./nix-env/rollback.md) + +These pages can be viewed offline: + +- `man nix-env-`. + + Example: `man nix-env-install` + +- `nix-env --help --` + + Example: `nix-env --help --install` # Selectors -Several commands, such as `nix-env -q` and `nix-env -i`, take a list of +Several commands, such as `nix-env --query ` and `nix-env --install `, take a list of arguments that specify the packages on which to operate. These are extended regular expressions that must match the entire name of the -package. (For details on regular expressions, see regex7.) The match is +package. (For details on regular expressions, see **regex**(7).) The match is case-sensitive. The regular expression can optionally be followed by a dash and a version number; if omitted, any version of the package will match. Here are some examples: @@ -60,833 +81,10 @@ Matches any package name containing the strings `firefox` or `chromium`. -# Common options - -This section lists the options that are common to all operations. These -options are allowed for every subcommand, though they may not always -have an effect. - - - `--file` / `-f` *path*\ - Specifies the Nix expression (designated below as the *active Nix - expression*) used by the `--install`, `--upgrade`, and `--query - --available` operations to obtain derivations. The default is - `~/.nix-defexpr`. - - If the argument starts with `http://` or `https://`, it is - interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must include a single - top-level directory containing at least a file named `default.nix`. - - - `--profile` / `-p` *path*\ - Specifies the profile to be used by those operations that operate on - a profile (designated below as the *active profile*). A profile is a - sequence of user environments called *generations*, one of which is - the *current generation*. - - - `--dry-run`\ - For the `--install`, `--upgrade`, `--uninstall`, - `--switch-generation`, `--delete-generations` and `--rollback` - operations, this flag will cause `nix-env` to print what *would* be - done if this flag had not been specified, without actually doing it. - - `--dry-run` also prints out which paths will be - [substituted](../glossary.md) (i.e., downloaded) and which paths - will be built from source (because no substitute is available). - - - `--system-filter` *system*\ - By default, operations such as `--query - --available` show derivations matching any platform. This option - allows you to use derivations for the specified platform *system*. - - - # Files - - `~/.nix-defexpr`\ - The source for the default Nix expressions used by the - `--install`, `--upgrade`, and `--query --available` operations to - obtain derivations. The `--file` option may be used to override - this default. - - If `~/.nix-defexpr` is a file, it is loaded as a Nix expression. If - the expression is a set, it is used as the default Nix expression. - If the expression is a function, an empty set is passed as argument - and the return value is used as the default Nix expression. - - If `~/.nix-defexpr` is a directory containing a `default.nix` file, - that file is loaded as in the above paragraph. - - If `~/.nix-defexpr` is a directory without a `default.nix` file, - then its contents (both files and subdirectories) are loaded as Nix - expressions. The expressions are combined into a single set, each - expression under an attribute with the same name as the original - file or subdirectory. - - For example, if `~/.nix-defexpr` contains two files, `foo.nix` and - `bar.nix`, then the default Nix expression will essentially be - - ```nix - { - foo = import ~/.nix-defexpr/foo.nix; - bar = import ~/.nix-defexpr/bar.nix; - } - ``` - - The file `manifest.nix` is always ignored. Subdirectories without a - `default.nix` file are traversed recursively in search of more Nix - expressions, but the names of these intermediate directories are not - added to the attribute paths of the default Nix expression. - - The command `nix-channel` places symlinks to the downloaded Nix - expressions from each subscribed channel in this directory. - - - `~/.nix-profile`\ - A symbolic link to the user's current profile. By default, this - symlink points to `prefix/var/nix/profiles/default`. The `PATH` - environment variable should include `~/.nix-profile/bin` for the - user environment to be visible to the user. - -# Operation `--install` - -## Synopsis - -`nix-env` {`--install` | `-i`} *args…* - [{`--prebuilt-only` | `-b`}] - [{`--attr` | `-A`}] - [`--from-expression`] [`-E`] - [`--from-profile` *path*] - [`--preserve-installed` | `-P`] - [`--remove-all` | `-r`] - -## Description - -The install operation creates a new user environment, based on the -current generation of the active profile, to which a set of store paths -described by *args* is added. The arguments *args* map to store paths in -a number of possible ways: - - - By default, *args* is a set of derivation names denoting derivations - in the active Nix expression. These are realised, and the resulting - output paths are installed. Currently installed derivations with a - name equal to the name of a derivation being added are removed - unless the option `--preserve-installed` is specified. - - If there are multiple derivations matching a name in *args* that - have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the - derivation with the highest *priority* is used. A derivation can - define a priority by declaring the `meta.priority` attribute. This - attribute should be a number, with a higher value denoting a lower - priority. The default priority is `0`. - - If there are multiple matching derivations with the same priority, - then the derivation with the highest version will be installed. - - You can force the installation of multiple derivations with the same - name by being specific about the versions. For instance, `nix-env -i - gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will - probably cause a user environment conflict\!). - - - If `--attr` (`-A`) is specified, the arguments are *attribute - paths* that select attributes from the top-level Nix - expression. This is faster than using derivation names and - unambiguous. To find out the attribute paths of available - packages, use `nix-env -qaP`. - - - If `--from-profile` *path* is given, *args* is a set of names - denoting installed store paths in the profile *path*. This is an - easy way to copy user environment elements from one profile to - another. - - - If `--from-expression` is given, *args* are Nix - [functions](../expressions/language-constructs.md#functions) - that are called with the active Nix expression as their single - argument. The derivations returned by those function calls are - installed. This allows derivations to be specified in an - unambiguous way, which is necessary if there are multiple - derivations with the same name. - - - If *args* are store derivations, then these are - [realised](nix-store.md#operation---realise), and the resulting output paths - are installed. - - - If *args* are store paths that are not store derivations, then these - are [realised](nix-store.md#operation---realise) and installed. - - - By default all outputs are installed for each derivation. That can - be reduced by setting `meta.outputsToInstall`. - -## Flags - - - `--prebuilt-only` / `-b`\ - Use only derivations for which a substitute is registered, i.e., - there is a pre-built binary available that can be downloaded in lieu - of building the derivation. Thus, no packages will be built from - source. - - - `--preserve-installed`; `-P`\ - Do not remove derivations with a name matching one of the - derivations being installed. Usually, trying to have two versions of - the same package installed in the same generation of a profile will - lead to an error in building the generation, due to file name - clashes between the two versions. However, this is not the case for - all packages. - - - `--remove-all`; `-r`\ - Remove all previously installed packages first. This is equivalent - to running `nix-env -e '.*'` first, except that everything happens - in a single transaction. - -## Examples - -To install a package using a specific attribute path from the active Nix expression: - -```console -$ nix-env -iA gcc40mips -installing `gcc-4.0.2' -$ nix-env -iA xorg.xorgserver -installing `xorg-server-1.2.0' -``` - -To install a specific version of `gcc` using the derivation name: - -```console -$ nix-env --install gcc-3.3.2 -installing `gcc-3.3.2' -uninstalling `gcc-3.1' -``` - -Using attribute path for selecting a package is preferred, -as it is much faster and there will not be multiple matches. - -Note the previously installed version is removed, since -`--preserve-installed` was not specified. - -To install an arbitrary version: - -```console -$ nix-env --install gcc -installing `gcc-3.3.2' -``` - -To install all derivations in the Nix expression `foo.nix`: - -```console -$ nix-env -f ~/foo.nix -i '.*' -``` - -To copy the store path with symbolic name `gcc` from another profile: - -```console -$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc -``` - -To install a specific store derivation (typically created by -`nix-instantiate`): - -```console -$ nix-env -i /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv -``` - -To install a specific output path: - -```console -$ nix-env -i /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3 -``` - -To install from a Nix expression specified on the command-line: - -```console -$ nix-env -f ./foo.nix -i -E \ - 'f: (f {system = "i686-linux";}).subversionWithJava' -``` - -I.e., this evaluates to `(f: (f {system = -"i686-linux";}).subversionWithJava) (import ./foo.nix)`, thus selecting -the `subversionWithJava` attribute from the set returned by calling the -function defined in `./foo.nix`. - -A dry-run tells you which paths will be downloaded or built from source: - -```console -$ nix-env -f '' -iA hello --dry-run -(dry run; not doing anything) -installing ‘hello-2.10’ -this path will be fetched (0.04 MiB download, 0.19 MiB unpacked): - /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10 - ... -``` - -To install Firefox from the latest revision in the Nixpkgs/NixOS 14.12 -channel: - -```console -$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox -``` - -# Operation `--upgrade` - -## Synopsis - -`nix-env` {`--upgrade` | `-u`} *args* - [`--lt` | `--leq` | `--eq` | `--always`] - [{`--prebuilt-only` | `-b`}] - [{`--attr` | `-A`}] - [`--from-expression`] [`-E`] - [`--from-profile` *path*] - [`--preserve-installed` | `-P`] - -## Description - -The upgrade operation creates a new user environment, based on the -current generation of the active profile, in which all store paths are -replaced for which there are newer versions in the set of paths -described by *args*. Paths for which there are no newer versions are -left untouched; this is not an error. It is also not an error if an -element of *args* matches no installed derivations. - -For a description of how *args* is mapped to a set of store paths, see -[`--install`](#operation---install). If *args* describes multiple -store paths with the same symbolic name, only the one with the highest -version is installed. - -## Flags - - - `--lt`\ - Only upgrade a derivation to newer versions. This is the default. - - - `--leq`\ - In addition to upgrading to newer versions, also “upgrade” to - derivations that have the same version. Version are not a unique - identification of a derivation, so there may be many derivations - that have the same version. This flag may be useful to force - “synchronisation” between the installed and available derivations. - - - `--eq`\ - *Only* “upgrade” to derivations that have the same version. This may - not seem very useful, but it actually is, e.g., when there is a new - release of Nixpkgs and you want to replace installed applications - with the same versions built against newer dependencies (to reduce - the number of dependencies floating around on your system). - - - `--always`\ - In addition to upgrading to newer versions, also “upgrade” to - derivations that have the same or a lower version. I.e., derivations - may actually be downgraded depending on what is available in the - active Nix expression. - -For the other flags, see `--install`. - -## Examples - -```console -$ nix-env --upgrade -A nixpkgs.gcc -upgrading `gcc-3.3.1' to `gcc-3.4' -``` - -When there are no updates available, nothing will happen: - -```console -$ nix-env --upgrade -A nixpkgs.pan -``` - -Using `-A` is preferred when possible, as it is faster and unambiguous but -it is also possible to upgrade to a specific version by matching the derivation name: - -```console -$ nix-env -u gcc-3.3.2 --always -upgrading `gcc-3.4' to `gcc-3.3.2' -``` - -To try to upgrade everything -(matching packages based on the part of the derivation name without version): - -```console -$ nix-env -u -upgrading `hello-2.1.2' to `hello-2.1.3' -upgrading `mozilla-1.2' to `mozilla-1.4' -``` - -## Versions - -The upgrade operation determines whether a derivation `y` is an upgrade -of a derivation `x` by looking at their respective `name` attributes. -The names (e.g., `gcc-3.3.1` are split into two parts: the package name -(`gcc`), and the version (`3.3.1`). The version part starts after the -first dash not followed by a letter. `x` is considered an upgrade of `y` -if their package names match, and the version of `y` is higher than that -of `x`. - -The versions are compared by splitting them into contiguous components -of numbers and letters. E.g., `3.3.1pre5` is split into `[3, 3, 1, -"pre", 5]`. These lists are then compared lexicographically (from left -to right). Corresponding components `a` and `b` are compared as follows. -If they are both numbers, integer comparison is used. If `a` is an empty -string and `b` is a number, `a` is considered less than `b`. The special -string component `pre` (for *pre-release*) is considered to be less than -other components. String components are considered less than number -components. Otherwise, they are compared lexicographically (i.e., using -case-sensitive string comparison). - -This is illustrated by the following examples: - - 1.0 < 2.3 - 2.1 < 2.3 - 2.3 = 2.3 - 2.5 > 2.3 - 3.1 > 2.3 - 2.3.1 > 2.3 - 2.3.1 > 2.3a - 2.3pre1 < 2.3 - 2.3pre3 < 2.3pre12 - 2.3a < 2.3c - 2.3pre1 < 2.3c - 2.3pre1 < 2.3q - -# Operation `--uninstall` - -## Synopsis - -`nix-env` {`--uninstall` | `-e`} *drvnames…* - -## Description - -The uninstall operation creates a new user environment, based on the -current generation of the active profile, from which the store paths -designated by the symbolic names *drvnames* are removed. - -## Examples - -```console -$ nix-env --uninstall gcc -$ nix-env -e '.*' (remove everything) -``` - -# Operation `--set` - -## Synopsis - -`nix-env` `--set` *drvname* - -## Description - -The `--set` operation modifies the current generation of a profile so -that it contains exactly the specified derivation, and nothing else. - -## Examples - -The following updates a profile such that its current generation will -contain just Firefox: - -```console -$ nix-env -p /nix/var/nix/profiles/browser --set firefox -``` - -# Operation `--set-flag` - -## Synopsis - -`nix-env` `--set-flag` *name* *value* *drvnames* - -## Description - -The `--set-flag` operation allows meta attributes of installed packages -to be modified. There are several attributes that can be usefully -modified, because they affect the behaviour of `nix-env` or the user -environment build script: - - - `priority` can be changed to resolve filename clashes. The user - environment build script uses the `meta.priority` attribute of - derivations to resolve filename collisions between packages. Lower - priority values denote a higher priority. For instance, the GCC - wrapper package and the Binutils package in Nixpkgs both have a file - `bin/ld`, so previously if you tried to install both you would get a - collision. Now, on the other hand, the GCC wrapper declares a higher - priority than Binutils, so the former’s `bin/ld` is symlinked in the - user environment. - - - `keep` can be set to `true` to prevent the package from being - upgraded or replaced. This is useful if you want to hang on to an - older version of a package. - - - `active` can be set to `false` to “disable” the package. That is, no - symlinks will be generated to the files of the package, but it - remains part of the profile (so it won’t be garbage-collected). It - can be set back to `true` to re-enable the package. - -## Examples - -To prevent the currently installed Firefox from being upgraded: - -```console -$ nix-env --set-flag keep true firefox -``` - -After this, `nix-env -u` will ignore Firefox. - -To disable the currently installed Firefox, then install a new Firefox -while the old remains part of the profile: - -```console -$ nix-env -q -firefox-2.0.0.9 (the current one) - -$ nix-env --preserve-installed -i firefox-2.0.0.11 -installing `firefox-2.0.0.11' -building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment' -collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox' - and `/nix/store/...-firefox-2.0.0.9/bin/firefox'. -(i.e., can’t have two active at the same time) - -$ nix-env --set-flag active false firefox -setting flag on `firefox-2.0.0.9' - -$ nix-env --preserve-installed -i firefox-2.0.0.11 -installing `firefox-2.0.0.11' - -$ nix-env -q -firefox-2.0.0.11 (the enabled one) -firefox-2.0.0.9 (the disabled one) -``` - -To make files from `binutils` take precedence over files from `gcc`: - -```console -$ nix-env --set-flag priority 5 binutils -$ nix-env --set-flag priority 10 gcc -``` - -# Operation `--query` - -## Synopsis - -`nix-env` {`--query` | `-q`} *names…* - [`--installed` | `--available` | `-a`] - [{`--status` | `-s`}] - [{`--attr-path` | `-P`}] - [`--no-name`] - [{`--compare-versions` | `-c`}] - [`--system`] - [`--drv-path`] - [`--out-path`] - [`--description`] - [`--meta`] - [`--xml`] - [`--json`] - [{`--prebuilt-only` | `-b`}] - [{`--attr` | `-A`} *attribute-path*] - -## Description - -The query operation displays information about either the store paths -that are installed in the current generation of the active profile -(`--installed`), or the derivations that are available for installation -in the active Nix expression (`--available`). It only prints information -about derivations whose symbolic name matches one of *names*. - -The derivations are sorted by their `name` attributes. - -## Source selection - -The following flags specify the set of things on which the query -operates. - - - `--installed`\ - The query operates on the store paths that are installed in the - current generation of the active profile. This is the default. - - - `--available`; `-a`\ - The query operates on the derivations that are available in the - active Nix expression. - -## Queries - -The following flags specify what information to display about the -selected derivations. Multiple flags may be specified, in which case the -information is shown in the order given here. Note that the name of the -derivation is shown unless `--no-name` is specified. - - - `--xml`\ - Print the result in an XML representation suitable for automatic - processing by other tools. The root element is called `items`, which - contains a `item` element for each available or installed - derivation. The fields discussed below are all stored in attributes - of the `item` elements. - - - `--json`\ - Print the result in a JSON representation suitable for automatic - processing by other tools. - - - `--prebuilt-only` / `-b`\ - Show only derivations for which a substitute is registered, i.e., - there is a pre-built binary available that can be downloaded in lieu - of building the derivation. Thus, this shows all packages that - probably can be installed quickly. - - - `--status`; `-s`\ - Print the *status* of the derivation. The status consists of three - characters. The first is `I` or `-`, indicating whether the - derivation is currently installed in the current generation of the - active profile. This is by definition the case for `--installed`, - but not for `--available`. The second is `P` or `-`, indicating - whether the derivation is present on the system. This indicates - whether installation of an available derivation will require the - derivation to be built. The third is `S` or `-`, indicating whether - a substitute is available for the derivation. - - - `--attr-path`; `-P`\ - Print the *attribute path* of the derivation, which can be used to - unambiguously select it using the `--attr` option available in - commands that install derivations like `nix-env --install`. This - option only works together with `--available` - - - `--no-name`\ - Suppress printing of the `name` attribute of each derivation. - - - `--compare-versions` / `-c`\ - Compare installed versions to available versions, or vice versa (if - `--available` is given). This is useful for quickly seeing whether - upgrades for installed packages are available in a Nix expression. A - column is added with the following meaning: - - - `<` *version*\ - A newer version of the package is available or installed. - - - `=` *version*\ - At most the same version of the package is available or - installed. - - - `>` *version*\ - Only older versions of the package are available or installed. - - - `- ?`\ - No version of the package is available or installed. - - - `--system`\ - Print the `system` attribute of the derivation. - - - `--drv-path`\ - Print the path of the store derivation. - - - `--out-path`\ - Print the output path of the derivation. - - - `--description`\ - Print a short (one-line) description of the derivation, if - available. The description is taken from the `meta.description` - attribute of the derivation. - - - `--meta`\ - Print all of the meta-attributes of the derivation. This option is - only available with `--xml` or `--json`. - -## Examples - -To show installed packages: - -```console -$ nix-env -q -bison-1.875c -docbook-xml-4.2 -firefox-1.0.4 -MPlayer-1.0pre7 -ORBit2-2.8.3 -… -``` - -To show available packages: - -```console -$ nix-env -qa -firefox-1.0.7 -GConf-2.4.0.1 -MPlayer-1.0pre7 -ORBit2-2.8.3 -… -``` - -To show the status of available packages: - -```console -$ nix-env -qas --P- firefox-1.0.7 (not installed but present) ---S GConf-2.4.0.1 (not present, but there is a substitute for fast installation) ---S MPlayer-1.0pre3 (i.e., this is not the installed MPlayer, even though the version is the same!) -IP- ORBit2-2.8.3 (installed and by definition present) -… -``` - -To show available packages in the Nix expression `foo.nix`: - -```console -$ nix-env -f ./foo.nix -qa -foo-1.2.3 -``` - -To compare installed versions to what’s available: - -```console -$ nix-env -qc -... -acrobat-reader-7.0 - ? (package is not available at all) -autoconf-2.59 = 2.59 (same version) -firefox-1.0.4 < 1.0.7 (a more recent version is available) -... -``` - -To show all packages with “`zip`” in the name: - -```console -$ nix-env -qa '.*zip.*' -bzip2-1.0.6 -gzip-1.6 -zip-3.0 -… -``` - -To show all packages with “`firefox`” or “`chromium`” in the name: - -```console -$ nix-env -qa '.*(firefox|chromium).*' -chromium-37.0.2062.94 -chromium-beta-38.0.2125.24 -firefox-32.0.3 -firefox-with-plugins-13.0.1 -… -``` - -To show all packages in the latest revision of the Nixpkgs repository: - -```console -$ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa -``` - -# Operation `--switch-profile` - -## Synopsis - -`nix-env` {`--switch-profile` | `-S`} *path* - -## Description - -This operation makes *path* the current profile for the user. That is, -the symlink `~/.nix-profile` is made to point to *path*. - -## Examples - -```console -$ nix-env -S ~/my-profile -``` - -# Operation `--list-generations` - -## Synopsis - -`nix-env` `--list-generations` - -## Description - -This operation print a list of all the currently existing generations -for the active profile. These may be switched to using the -`--switch-generation` operation. It also prints the creation date of the -generation, and indicates the current generation. - -## Examples - -```console -$ nix-env --list-generations - 95 2004-02-06 11:48:24 - 96 2004-02-06 11:49:01 - 97 2004-02-06 16:22:45 - 98 2004-02-06 16:24:33 (current) -``` - -# Operation `--delete-generations` - -## Synopsis - -`nix-env` `--delete-generations` *generations* - -## Description - -This operation deletes the specified generations of the current profile. -The generations can be a list of generation numbers, the special value -`old` to delete all non-current generations, a value such as `30d` to -delete all generations older than the specified number of days (except -for the generation that was active at that point in time), or a value -such as `+5` to keep the last `5` generations ignoring any newer than -current, e.g., if `30` is the current generation `+5` will delete -generation `25` and all older generations. Periodically deleting old -generations is important to make garbage collection effective. - -## Examples - -```console -$ nix-env --delete-generations 3 4 8 -``` - -```console -$ nix-env --delete-generations +5 -``` - -```console -$ nix-env --delete-generations 30d -``` - -```console -$ nix-env -p other_profile --delete-generations old -``` - -# Operation `--switch-generation` - -## Synopsis - -`nix-env` {`--switch-generation` | `-G`} *generation* - -## Description - -This operation makes generation number *generation* the current -generation of the active profile. That is, if the `profile` is the path -to the active profile, then the symlink `profile` is made to point to -`profile-generation-link`, which is in turn a symlink to the actual user -environment in the Nix store. - -Switching will fail if the specified generation does not exist. - -## Examples - -```console -$ nix-env -G 42 -switching from generation 50 to 42 -``` - -# Operation `--rollback` - -## Synopsis - -`nix-env` `--rollback` - -## Description - -This operation switches to the “previous” generation of the active -profile, that is, the highest numbered generation lower than the current -generation, if it exists. It is just a convenience wrapper around -`--list-generations` and `--switch-generation`. - -## Examples - -```console -$ nix-env --rollback -switching from generation 92 to 91 -``` - -```console -$ nix-env --rollback -error: no generation older than the current (91) exists -``` +`nix-env` operates on the following files. -# Environment variables +{{#include ./files/default-nix-expression.md}} - - `NIX_PROFILE`\ - Location of the Nix profile. Defaults to the target of the symlink - `~/.nix-profile`, if it exists, or `/nix/var/nix/profiles/default` - otherwise. +{{#include ./files/profiles.md}} diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-hash.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-hash.md --- nix-2.8.0/doc/manual/src/command-ref/nix-hash.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-hash.md 2023-06-06 10:55:23.000000000 +0000 @@ -6,9 +6,7 @@ `nix-hash` [`--flat`] [`--base32`] [`--truncate`] [`--type` *hashAlgo*] *path…* -`nix-hash` `--to-base16` *hash…* - -`nix-hash` `--to-base32` *hash…* +`nix-hash` [`--to-base16`|`--to-base32`|`--to-base64`|`--to-sri`] [`--type` *hashAlgo*] *hash…* # Description @@ -23,7 +21,7 @@ the file system tree rooted at the path. This allows directories and symlinks to be hashed as well as regular files. The dump is in the *NAR format* produced by [`nix-store ---dump`](nix-store.md#operation---dump). Thus, `nix-hash path` +--dump`](@docroot@/command-ref/nix-store/dump.md). Thus, `nix-hash path` yields the same cryptographic hash as `nix-store --dump path | md5sum`. @@ -35,11 +33,23 @@ The result is identical to that produced by the GNU commands `md5sum` and `sha1sum`. + - `--base16`\ + Print the hash in a hexadecimal representation (default). + - `--base32`\ Print the hash in a base-32 representation rather than hexadecimal. This base-32 representation is more compact and can be used in Nix expressions (such as in calls to `fetchurl`). + - `--base64`\ + Similar to --base32, but print the hash in a base-64 representation, + which is more compact than the base-32 one. + + - `--sri`\ + Print the hash in SRI format with base-64 encoding. + The type of hash algorithm will be prepended to the hash string, + followed by a hyphen (-) and the base-64 hash body. + - `--truncate`\ Truncate hashes longer than 160 bits (such as SHA-256) to 160 bits. @@ -55,6 +65,14 @@ Don’t hash anything, but convert the hexadecimal hash representation *hash* to base-32. + - `--to-base64`\ + Don’t hash anything, but convert the hexadecimal hash representation + *hash* to base-64. + + - `--to-sri`\ + Don’t hash anything, but convert the hexadecimal hash representation + *hash* to SRI. + # Examples Computing the same hash as `nix-prefetch-url`: @@ -81,9 +99,18 @@ $ nix-hash --type sha1 test/ e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 +$ nix-hash --type sha1 --base16 test/ +e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 + $ nix-hash --type sha1 --base32 test/ nvd61k9nalji1zl9rrdfmsmvyyjqpzg4 +$ nix-hash --type sha1 --base64 test/ +5P2Lpfe76upazon+ECVVNs1g2rY= + +$ nix-hash --type sha1 --sri test/ +sha1-5P2Lpfe76upazon+ECVVNs1g2rY= + $ nix-hash --type sha256 --flat test/ error: reading file `test/': Is a directory @@ -91,7 +118,7 @@ 5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 ``` -Converting between hexadecimal and base-32: +Converting between hexadecimal, base-32, base-64, and SRI: ```console $ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 @@ -99,4 +126,13 @@ $ nix-hash --type sha1 --to-base16 nvd61k9nalji1zl9rrdfmsmvyyjqpzg4 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 + +$ nix-hash --type sha1 --to-base64 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 +5P2Lpfe76upazon+ECVVNs1g2rY= + +$ nix-hash --type sha1 --to-sri nvd61k9nalji1zl9rrdfmsmvyyjqpzg4 +sha1-5P2Lpfe76upazon+ECVVNs1g2rY= + +$ nix-hash --to-base16 sha1-5P2Lpfe76upazon+ECVVNs1g2rY= +e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 ``` diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-instantiate.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-instantiate.md --- nix-2.8.0/doc/manual/src/command-ref/nix-instantiate.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-instantiate.md 2023-06-06 10:55:23.000000000 +0000 @@ -17,13 +17,14 @@ # Description -The command `nix-instantiate` generates [store -derivations](../glossary.md) from (high-level) Nix expressions. It -evaluates the Nix expressions in each of *files* (which defaults to +The command `nix-instantiate` produces [store derivation]s from (high-level) Nix expressions. +It evaluates the Nix expressions in each of *files* (which defaults to *./default.nix*). Each top-level expression should evaluate to a derivation, a list of derivations, or a set of derivations. The paths of the resulting store derivations are printed on standard output. +[store derivation]: ../glossary.md#gloss-store-derivation + If *files* is the character `-`, then a Nix expression will be read from standard input. @@ -51,7 +52,7 @@ - `--strict`\ When used with `--eval`, recursively evaluate list elements and attributes. Normally, such sub-expressions are left unevaluated - (since the Nix expression language is lazy). + (since the Nix language is lazy). > **Warning** > @@ -66,7 +67,7 @@ When used with `--eval`, print the resulting value as an XML representation of the abstract syntax tree rather than as an ATerm. The schema is the same as that used by the [`toXML` - built-in](../expressions/builtins.md). + built-in](../language/builtins.md). - `--read-write-mode`\ When used with `--eval`, perform evaluation in read/write mode so @@ -75,18 +76,19 @@ this option is not enabled, there may be uninstantiated store paths in the final output. - +{{#include ./opt-common.md}} + +{{#include ./env-common.md}} # Examples -Instantiating store derivations from a Nix expression, and building them -using `nix-store`: +Instantiate [store derivation]s from a Nix expression, and build them using `nix-store`: ```console $ nix-instantiate test.nix (instantiate) /nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv -$ nix-store -r $(nix-instantiate test.nix) (build) +$ nix-store --realise $(nix-instantiate test.nix) (build) ... /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 (output path) @@ -98,30 +100,30 @@ You can also give a Nix expression on the command line: ```console -$ nix-instantiate -E 'with import { }; hello' +$ nix-instantiate --expr 'with import { }; hello' /nix/store/j8s4zyv75a724q38cb0r87rlczaiag4y-hello-2.8.drv ``` This is equivalent to: ```console -$ nix-instantiate '' -A hello +$ nix-instantiate '' --attr hello ``` Parsing and evaluating Nix expressions: ```console -$ nix-instantiate --parse -E '1 + 2' +$ nix-instantiate --parse --expr '1 + 2' 1 + 2 ``` ```console -$ nix-instantiate --eval -E '1 + 2' +$ nix-instantiate --eval --expr '1 + 2' 3 ``` ```console -$ nix-instantiate --eval --xml -E '1 + 2' +$ nix-instantiate --eval --xml --expr '1 + 2' @@ -131,7 +133,7 @@ The difference between non-strict and strict evaluation: ```console -$ nix-instantiate --eval --xml -E 'rec { x = "foo"; y = x; }' +$ nix-instantiate --eval --xml --expr 'rec { x = "foo"; y = x; }' ... @@ -146,7 +148,7 @@ attempt to show non-normal forms). ```console -$ nix-instantiate --eval --xml --strict -E 'rec { x = "foo"; y = x; }' +$ nix-instantiate --eval --xml --strict --expr 'rec { x = "foo"; y = x; }' ... diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-shell.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-shell.md --- nix-2.8.0/doc/manual/src/command-ref/nix-shell.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-shell.md 2023-06-06 10:55:23.000000000 +0000 @@ -15,6 +15,12 @@ [`--keep` *name*] {{`--packages` | `-p`} {*packages* | *expressions*} … | [*path*]} +# Disambiguation + +This man page describes the command `nix-shell`, which is distinct from `nix +shell`. For documentation on the latter, run `nix shell --help` or see `man +nix3-shell`. + # Description The command `nix-shell` will build the dependencies of the specified @@ -83,7 +89,7 @@ - `--packages` / `-p` *packages*…\ Set up an environment in which the specified packages are present. The command line arguments are interpreted as attribute names inside - the Nix Packages collection. Thus, `nix-shell -p libjpeg openjdk` + the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` will start a shell in which the packages denoted by the attribute names `libjpeg` and `openjdk` are present. @@ -95,7 +101,7 @@ When a `--pure` shell is started, keep the listed environment variables. -The following common options are supported: +{{#include ./opt-common.md}} # Environment variables @@ -104,15 +110,18 @@ `bash` found in ``, falling back to the `bash` found in `PATH` if not found. +{{#include ./env-common.md}} + # Examples To build the dependencies of the package Pan, and start an interactive shell in which to build it: ```console -$ nix-shell '' -A pan +$ nix-shell '' --attr pan [nix-shell]$ eval ${unpackPhase:-unpackPhase} -[nix-shell]$ cd pan-* +[nix-shell]$ cd $sourceRoot +[nix-shell]$ eval ${patchPhase:-patchPhase} [nix-shell]$ eval ${configurePhase:-configurePhase} [nix-shell]$ eval ${buildPhase:-buildPhase} [nix-shell]$ ./pan/gui/pan @@ -128,7 +137,7 @@ initialisation of the interactive shell: ```console -$ nix-shell '' -A pan --pure \ +$ nix-shell '' --attr pan --pure \ --command 'export NIX_DEBUG=1; export NIX_CORES=8; return' ``` @@ -137,13 +146,13 @@ packages `sqlite` and `libX11`: ```console -$ nix-shell -E 'with import { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""' +$ nix-shell --expr 'with import { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""' ``` A shorter way to do the same is: ```console -$ nix-shell -p sqlite xorg.libX11 +$ nix-shell --packages sqlite xorg.libX11 [nix-shell]$ echo $NIX_LDFLAGS … -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib … ``` @@ -153,7 +162,7 @@ following is also legal: ```console -$ nix-shell -p sqlite 'git.override { withManual = false; }' +$ nix-shell --packages sqlite 'git.override { withManual = false; }' ``` The `-p` flag looks up Nixpkgs in the Nix search path. You can override @@ -162,7 +171,7 @@ Nixpkgs: ```console -$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz +$ nix-shell --packages pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz [nix-shell:~]$ pan --version Pan 0.139 @@ -176,7 +185,7 @@ ```bash #! /usr/bin/env nix-shell -#! nix-shell -i real-interpreter -p packages +#! nix-shell -i real-interpreter --packages packages ``` where *real-interpreter* is the “real” script interpreter that will be @@ -193,7 +202,7 @@ ```python #! /usr/bin/env nix-shell -#! nix-shell -i python -p python pythonPackages.prettytable +#! nix-shell -i python --packages python pythonPackages.prettytable import prettytable @@ -208,7 +217,7 @@ ```perl #! /usr/bin/env nix-shell -#! nix-shell -i perl -p perl perlPackages.HTMLTokeParserSimple perlPackages.LWP +#! nix-shell -i perl --packages perl perlPackages.HTMLTokeParserSimple perlPackages.LWP use HTML::TokeParser::Simple; @@ -226,7 +235,7 @@ ```bash #! /usr/bin/env nix-shell -#! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix-shell -i bash --packages "terraform.withPlugins (plugins: [ plugins.openstack ])" terraform apply ``` @@ -242,7 +251,7 @@ ```haskell #! /usr/bin/env nix-shell -#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" +#! nix-shell -i runghc --packages "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz import Network.Curl.Download diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/add-fixed.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/add-fixed.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/add-fixed.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/add-fixed.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,35 @@ +# Name + +`nix-store --add-fixed` - add paths to store using given hashing algorithm + +## Synopsis + +`nix-store` `--add-fixed` [`--recursive`] *algorithm* *paths…* + +## Description + +The operation `--add-fixed` adds the specified paths to the Nix store. +Unlike `--add` paths are registered using the specified hashing +algorithm, resulting in the same output path as a fixed-output +derivation. This can be used for sources that are not available from a +public url or broke since the download expression was written. + +This operation has the following options: + + - `--recursive`\ + Use recursive instead of flat hashing mode, used when adding + directories to the store. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +## Example + +```console +$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz +/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/add.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/add.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/add.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/add.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,25 @@ +# Name + +`nix-store --add` - add paths to Nix store + +# Synopsis + +`nix-store` `--add` *paths…* + +# Description + +The operation `--add` adds the specified paths to the Nix store. It +prints the resulting paths in the Nix store on standard output. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Example + +```console +$ nix-store --add ./foo.c +/nix/store/m7lrha58ph6rcnv109yzx1nk1cj7k7zf-foo.c +``` diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/delete.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/delete.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/delete.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/delete.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,33 @@ +# Name + +`nix-store --delete` - delete store paths + +# Synopsis + +`nix-store` `--delete` [`--ignore-liveness`] *paths…* + +# Description + +The operation `--delete` deletes the store paths *paths* from the Nix +store, but only if it is safe to do so; that is, when the path is not +reachable from a root of the garbage collector. This means that you can +only delete paths that would also be deleted by `nix-store --gc`. Thus, +`--delete` is a more targeted version of `--gc`. + +With the option `--ignore-liveness`, reachability from the roots is +ignored. However, the path still won’t be deleted if there are other +paths in the store that refer to it (i.e., depend on it). + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Example + +```console +$ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4 +0 bytes freed (0.00 MiB) +error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive +``` diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/dump-db.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/dump-db.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/dump-db.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/dump-db.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,26 @@ +# Name + +`nix-store --dump-db` - export Nix database + +# Synopsis + +`nix-store` `--dump-db` [*paths…*] + +# Description + +The operation `--dump-db` writes a dump of the Nix database to standard +output. It can be loaded into an empty Nix store using `--load-db`. This +is useful for making backups and when migrating to different database +schemas. + +By default, `--dump-db` will dump the entire Nix database. When one or +more store paths is passed, only the subset of the Nix database for +those store paths is dumped. As with `--export`, the user is responsible +for passing all the store paths for a closure. See `--export` for an +example. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/dump.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/dump.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/dump.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/dump.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,40 @@ +# Name + +`nix-store --dump` - write a single path to a Nix Archive + +## Synopsis + +`nix-store` `--dump` *path* + +## Description + +The operation `--dump` produces a NAR (Nix ARchive) file containing the +contents of the file system tree rooted at *path*. The archive is +written to standard output. + +A NAR archive is like a TAR or Zip archive, but it contains only the +information that Nix considers important. For instance, timestamps are +elided because all files in the Nix store have their timestamp set to 0 +anyway. Likewise, all permissions are left out except for the execute +bit, because all files in the Nix store have 444 or 555 permission. + +Also, a NAR archive is *canonical*, meaning that “equal” paths always +produce the same NAR archive. For instance, directory entries are +always sorted so that the actual on-disk order doesn’t influence the +result. This means that the cryptographic hash of a NAR dump of a +path is usable as a fingerprint of the contents of the path. Indeed, +the hashes of store paths stored in Nix’s database (see `nix-store --query +--hash`) are SHA-256 hashes of the NAR dump of each store path. + +NAR archives support filenames of unlimited length and 64-bit file +sizes. They can contain regular files, directories, and symbolic links, +but not other types of files (such as device nodes). + +A Nix archive can be unpacked using `nix-store +--restore`. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/export.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/export.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/export.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/export.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,41 @@ +# Name + +`nix-store --export` - export store paths to a Nix Archive + +## Synopsis + +`nix-store` `--export` *paths…* + +## Description + +The operation `--export` writes a serialisation of the specified store +paths to standard output in a format that can be imported into another +Nix store with `nix-store --import`. This is like `nix-store +--dump`, except that the NAR archive produced by that command doesn’t +contain the necessary meta-information to allow it to be imported into +another Nix store (namely, the set of references of the path). + +This command does not produce a *closure* of the specified paths, so if +a store path references other store paths that are missing in the target +Nix store, the import will fail. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To copy a whole closure, do something +like: + +```console +$ nix-store --export $(nix-store --query --requisites paths) > out +``` + +To import the whole closure again, run: + +```console +$ nix-store --import < out +``` diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/gc.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/gc.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/gc.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/gc.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,72 @@ +# Name + +`nix-store --gc` - run garbage collection + +# Synopsis + +`nix-store` `--gc` [`--print-roots` | `--print-live` | `--print-dead`] [`--max-freed` *bytes*] + +# Description + +Without additional flags, the operation `--gc` performs a garbage +collection on the Nix store. That is, all paths in the Nix store not +reachable via file system references from a set of “roots”, are deleted. + +The following suboperations may be specified: + + - `--print-roots`\ + This operation prints on standard output the set of roots used by + the garbage collector. + + - `--print-live`\ + This operation prints on standard output the set of “live” store + paths, which are all the store paths reachable from the roots. Live + paths should never be deleted, since that would break consistency — + it would become possible that applications are installed that + reference things that are no longer present in the store. + + - `--print-dead`\ + This operation prints out on standard output the set of “dead” store + paths, which is just the opposite of the set of live paths: any path + in the store that is not live (with respect to the roots) is dead. + +By default, all unreachable paths are deleted. The following options +control what gets deleted and in what order: + + - `--max-freed` *bytes*\ + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. + +The behaviour of the collector is also influenced by the +`keep-outputs` and `keep-derivations` settings in the Nix +configuration file. + +By default, the collector prints the total number of freed bytes when it +finishes (or when it is interrupted). With `--print-dead`, it prints the +number of bytes that would be freed. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To delete all unreachable paths, just do: + +```console +$ nix-store --gc +deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv' +... +8825586 bytes freed (8.42 MiB) +``` + +To delete at least 100 MiBs of unreachable paths: + +```console +$ nix-store --gc --max-freed $((100 * 1024 * 1024)) +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/generate-binary-cache-key.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/generate-binary-cache-key.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/generate-binary-cache-key.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/generate-binary-cache-key.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,29 @@ +# Name + +`nix-store --generate-binary-cache-key` - generate key pair to use for a binary cache + +## Synopsis + +`nix-store` `--generate-binary-cache-key` *key-name* *secret-key-file* *public-key-file* + +## Description + +This command generates an [Ed25519 key pair](http://ed25519.cr.yp.to/) +that can be used to create a signed binary cache. It takes three +mandatory parameters: + +1. A key name, such as `cache.example.org-1`, that is used to look up + keys on the client when it verifies signatures. It can be anything, + but it’s suggested to use the host name of your cache (e.g. + `cache.example.org`) with a suffix denoting the number of the key + (to be incremented every time you need to revoke a key). + +2. The file name where the secret key is to be stored. + +3. The file name where the public key is to be stored. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/import.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/import.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/import.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/import.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,21 @@ +# Name + +`nix-store --import` - import Nix Archive into the store + +# Synopsis + +`nix-store` `--import` + +# Description + +The operation `--import` reads a serialisation of a set of store paths +produced by `nix-store --export` from standard input and adds those +store paths to the Nix store. Paths that already exist in the Nix store +are ignored. If a path refers to another path that doesn’t exist in the +Nix store, the import fails. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/load-db.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/load-db.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/load-db.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/load-db.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,18 @@ +# Name + +`nix-store --load-db` - import Nix database + +# Synopsis + +`nix-store` `--load-db` + +# Description + +The operation `--load-db` reads a dump of the Nix database created by +`--dump-db` from standard input and loads it into the Nix database. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/opt-common.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/opt-common.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/opt-common.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/opt-common.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,36 @@ +# Options + +The following options are allowed for all `nix-store` operations, but may not always have an effect. + +- [`--add-root`](#opt-add-root) *path* + + Causes the result of a realisation (`--realise` and + `--force-realise`) to be registered as a root of the garbage + collector. *path* will be created as a symlink to the resulting + store path. In addition, a uniquely named symlink to *path* will + be created in `/nix/var/nix/gcroots/auto/`. For instance, + + ```console + $ nix-store --add-root /home/eelco/bla/result --realise ... + + $ ls -l /nix/var/nix/gcroots/auto + lrwxrwxrwx 1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result + + $ ls -l /home/eelco/bla/result + lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r11343n6qd4...-f-spot-0.0.10 + ``` + + Thus, when `/home/eelco/bla/result` is removed, the GC root in the + `auto` directory becomes a dangling symlink and will be ignored by + the collector. + + > **Warning** + > + > Note that it is not possible to move or rename GC roots, since + > the symlink in the `auto` directory will still point to the old + > location. + + If there are multiple results, then multiple symlinks will be + created by sequentially numbering symlinks beyond the first one + (e.g., `foo`, `foo-2`, `foo-3`, and so on). + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/optimise.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/optimise.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/optimise.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/optimise.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,40 @@ +# Name + +`nix-store --optimise` - reduce disk space usage + +## Synopsis + +`nix-store` `--optimise` + +## Description + +The operation `--optimise` reduces Nix store disk space usage by finding +identical files in the store and hard-linking them to each other. It +typically reduces the size of the store by something like 25-35%. Only +regular files and symlinks are hard-linked in this manner. Files are +considered identical when they have the same NAR archive serialisation: +that is, regular files must have the same contents and permission +(executable or non-executable), and symlinks must have the same +contents. + +After completion, or when the command is interrupted, a report on the +achieved savings is printed on standard error. + +Use `-vv` or `-vvv` to get some progress indication. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +## Example + +```console +$ nix-store --optimise +hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' +... +541838819 bytes (516.74 MiB) freed by hard-linking 54143 files; +there are 114486 files with equal contents out of 215894 files in total +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/print-env.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/print-env.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/print-env.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/print-env.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,31 @@ +# Name + +`nix-store --print-env` - print the build environment of a derivation + +## Synopsis + +`nix-store` `--print-env` *drvpath* + +## Description + +The operation `--print-env` prints out the environment of a derivation +in a format that can be evaluated by a shell. The command line arguments +of the builder are placed in the variable `_args`. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +## Example + +```console +$ nix-store --print-env $(nix-instantiate '' -A firefox) +… +export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2' +export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv' +export system; system='x86_64-linux' +export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-builder.sh' +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/query.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/query.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/query.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/query.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,220 @@ +# Name + +`nix-store --query` - display information about store paths + +# Synopsis + +`nix-store` {`--query` | `-q`} + {`--outputs` | `--requisites` | `-R` | `--references` | + `--referrers` | `--referrers-closure` | `--deriver` | `-d` | + `--graph` | `--tree` | `--binding` *name* | `-b` *name* | `--hash` | + `--size` | `--roots`} + [`--use-output`] [`-u`] [`--force-realise`] [`-f`] + *paths…* + +# Description + +The operation `--query` displays various bits of information about the +store paths . The queries are described below. At most one query can be +specified. The default query is `--outputs`. + +The paths *paths* may also be symlinks from outside of the Nix store, to +the Nix store. In that case, the query is applied to the target of the +symlink. + +# Common query options + + - `--use-output`; `-u`\ + For each argument to the query that is a [store derivation], apply the + query to the output path of the derivation instead. + + - `--force-realise`; `-f`\ + Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). + +[store derivation]: @docroot@/glossary.md#gloss-store-derivation + +# Queries + + - `--outputs`\ + Prints out the [output paths] of the store + derivations *paths*. These are the paths that will be produced when + the derivation is built. + + [output paths]: ../../glossary.md#gloss-output-path + + - `--requisites`; `-R`\ + Prints out the [closure] of the store path *paths*. + + [closure]: ../../glossary.md#gloss-closure + + This query has one option: + + - `--include-outputs` + Also include the existing output paths of [store derivation]s, + and their closures. + + This query can be used to implement various kinds of deployment. A + *source deployment* is obtained by distributing the closure of a + store derivation. A *binary deployment* is obtained by distributing + the closure of an output path. A *cache deployment* (combined + source/binary deployment, including binaries of build-time-only + dependencies) is obtained by distributing the closure of a store + derivation and specifying the option `--include-outputs`. + + - `--references`\ + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) + + [references]: ../../glossary.md#gloss-reference + + - `--referrers`\ + Prints the set of *referrers* of the store paths *paths*, that is, + the store paths currently existing in the Nix store that refer to + one of *paths*. Note that contrary to the references, the set of + referrers is not constant; it can change as store paths are added or + removed. + + - `--referrers-closure`\ + Prints the closure of the set of store paths *paths* under the + referrers relation; that is, all store paths that directly or + indirectly refer to one of *paths*. These are all the path currently + in the Nix store that are dependent on *paths*. + + - `--deriver`; `-d`\ + Prints the [deriver] of the store paths *paths*. If + the path has no deriver (e.g., if it is a source file), or if the + deriver is not known (e.g., in the case of a binary-only + deployment), the string `unknown-deriver` is printed. + + [deriver]: ../../glossary.md#gloss-deriver + + - `--graph`\ + Prints the references graph of the store paths *paths* in the format + of the `dot` tool of AT\&T's [Graphviz + package](http://www.graphviz.org/). This can be used to visualise + dependency graphs. To obtain a build-time dependency graph, apply + this to a store derivation. To obtain a runtime dependency graph, + apply it to an output path. + + - `--tree`\ + Prints the references graph of the store paths *paths* as a nested + ASCII tree. References are ordered by descending closure size; this + tends to flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the graph. + + - `--graphml`\ + Prints the references graph of the store paths *paths* in the + [GraphML](http://graphml.graphdrawing.org/) file format. This can be + used to visualise dependency graphs. To obtain a build-time + dependency graph, apply this to a [store derivation]. To obtain a + runtime dependency graph, apply it to an output path. + + - `--binding` *name*; `-b` *name*\ + Prints the value of the attribute *name* (i.e., environment + variable) of the [store derivation]s *paths*. It is an error for a + derivation to not have the specified attribute. + + - `--hash`\ + Prints the SHA-256 hash of the contents of the store paths *paths* + (that is, the hash of the output of `nix-store --dump` on the given + paths). Since the hash is stored in the Nix database, this is a fast + operation. + + - `--size`\ + Prints the size in bytes of the contents of the store paths *paths* + — to be precise, the size of the output of `nix-store --dump` on + the given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + + - `--roots`\ + Prints the garbage collector roots that point, directly or + indirectly, at the store paths *paths*. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +Print the closure (runtime dependencies) of the `svn` program in the +current user environment: + +```console +$ nix-store --query --requisites $(which svn) +/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 +/nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4 +... +``` + +Print the build-time dependencies of `svn`: + +```console +$ nix-store --query --requisites $(nix-store --query --deriver $(which svn)) +/nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv +/nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh +/nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv +... lots of other paths ... +``` + +The difference with the previous example is that we ask the closure of +the derivation (`-qd`), not the closure of the output path that contains +`svn`. + +Show the build-time dependencies as a tree: + +```console +$ nix-store --query --tree $(nix-store --query --deriver $(which svn)) +/nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv ++---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh ++---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv +| +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash +| +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh +... +``` + +Show all paths that depend on the same OpenSSL library as `svn`: + +```console +$ nix-store --query --referrers $(nix-store --query --binding openssl $(nix-store --query --deriver $(which svn))) +/nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0 +/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 +/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3 +/nix/store/l51240xqsgg8a7yrbqdx1rfzyv6l26fx-lynx-2.8.5 +``` + +Show all paths that directly or indirectly depend on the Glibc (C +library) used by `svn`: + +```console +$ nix-store --query --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}') +/nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2 +/nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4 +... +``` + +Note that `ldd` is a command that prints out the dynamic libraries used +by an ELF executable. + +Make a picture of the runtime dependency graph of the current user +environment: + +```console +$ nix-store --query --graph ~/.nix-profile | dot -Tps > graph.ps +$ gv graph.ps +``` + +Show every garbage collector root that points to a store path that +depends on `svn`: + +```console +$ nix-store --query --roots $(which svn) +/nix/var/nix/profiles/default-81-link +/nix/var/nix/profiles/default-82-link +/home/eelco/.local/state/nix/profiles/profile-97-link +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/read-log.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/read-log.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/read-log.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/read-log.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,38 @@ +# Name + +`nix-store --read-log` - print build log + +# Synopsis + +`nix-store` {`--read-log` | `-l`} *paths…* + +# Description + +The operation `--read-log` prints the build log of the specified store +paths on standard output. The build log is whatever the builder of a +derivation wrote to standard output and standard error. If a store path +is not a derivation, the deriver of the store path is used. + +Build logs are kept in `/nix/var/log/nix/drvs`. However, there is no +guarantee that a build log is available for any particular store path. +For instance, if the path was downloaded as a pre-built binary through a +substitute, then the log is unavailable. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Example + +```console +$ nix-store --read-log $(which ktorrent) +building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1 +unpacking sources +unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz +ktorrent-2.2.1/ +ktorrent-2.2.1/NEWS +... +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/realise.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/realise.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/realise.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/realise.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,118 @@ +# Name + +`nix-store --realise` - realise specified store paths + +# Synopsis + +`nix-store` {`--realise` | `-r`} *paths…* [`--dry-run`] + +# Description + +The operation `--realise` essentially “builds” the specified store +paths. Realisation is a somewhat overloaded term: + + - If the store path is a *derivation*, realisation ensures that the + output paths of the derivation are [valid] (i.e., + the output path and its closure exist in the file system). This + can be done in several ways. First, it is possible that the + outputs are already valid, in which case we are done + immediately. Otherwise, there may be [substitutes] + that produce the outputs (e.g., by downloading them). Finally, the + outputs can be produced by running the build task described + by the derivation. + + - If the store path is not a derivation, realisation ensures that the + specified path is valid (i.e., it and its closure exist in the file + system). If the path is already valid, we are done immediately. + Otherwise, the path and any missing paths in its closure may be + produced through substitutes. If there are no (successful) + substitutes, realisation fails. + +[valid]: @docroot@/glossary.md#gloss-validity +[substitutes]: @docroot@/glossary.md#gloss-substitute + +The output path of each derivation is printed on standard output. (For +non-derivations argument, the argument itself is printed.) + +The following flags are available: + + - `--dry-run`\ + Print on standard error a description of what packages would be + built or downloaded, without actually performing the operation. + + - `--ignore-unknown`\ + If a non-derivation path does not have a substitute, then silently + ignore it. + + - `--check`\ + This option allows you to check whether a derivation is + deterministic. It rebuilds the specified derivation and checks + whether the result is bitwise-identical with the existing outputs, + printing an error if that’s not the case. The outputs of the + specified derivation must already exist. When used with `-K`, if an + output path is not identical to the corresponding output from the + previous build, the new output path is left in + `/nix/store/name.check.` + +Special exit codes: + + - `100`\ + Generic build failure, the builder process returned with a non-zero + exit code. + + - `101`\ + Build timeout, the build was aborted because it did not complete + within the specified `timeout`. + + - `102`\ + Hash mismatch, the build output was rejected because it does not + match the [`outputHash` attribute of the + derivation](@docroot@/language/advanced-attributes.md). + + - `104`\ + Not deterministic, the build succeeded in check mode but the + resulting output is not binary reproducible. + +With the `--keep-going` flag it's possible for multiple failures to +occur, in this case the 1xx status codes are or combined using binary +or. + + 1100100 + ^^^^ + |||`- timeout + ||`-- output hash mismatch + |`--- build failure + `---- not deterministic + + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +This operation is typically used to build [store derivation]s produced by +[`nix-instantiate`](@docroot@/command-ref/nix-instantiate.md): + +[store derivation]: @docroot@/glossary.md#gloss-store-derivation + +```console +$ nix-store --realise $(nix-instantiate ./test.nix) +/nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1 +``` + +This is essentially what [`nix-build`](@docroot@/command-ref/nix-build.md) does. + +To test whether a previously-built derivation is deterministic: + +```console +$ nix-build '' --attr hello --check -K +``` + +Use [`nix-store --read-log`](./read-log.md) to show the stderr and stdout of a build: + +```console +$ nix-store --read-log $(nix-instantiate ./test.nix) +``` diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/repair-path.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/repair-path.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/repair-path.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/repair-path.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,35 @@ +# Name + +`nix --repair-path` - re-download path from substituter + +# Synopsis + +`nix-store` `--repair-path` *paths…* + +# Description + +The operation `--repair-path` attempts to “repair” the specified paths +by redownloading them using the available substituters. If no +substitutes are available, then repair is not possible. + +> **Warning** +> +> During repair, there is a very small time window during which the old +> path (if it exists) is moved out of the way and replaced with the new +> path. If repair is interrupted in between, then the system may be left +> in a broken state (e.g., if the path contains a critical system +> component like the GNU C Library). + +# Example + +```console +$ nix-store --verify-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 +path `/nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13' was modified! + expected hash `2db57715ae90b7e31ff1f2ecb8c12ec1cc43da920efcbe3b22763f36a1861588', + got `481c5aa5483ebc97c20457bb8bca24deea56550d3985cda0027f67fe54b808e4' + +$ nix-store --repair-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 +fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'... +… +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/restore.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/restore.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/restore.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/restore.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,18 @@ +# Name + +`nix-store --restore` - extract a Nix archive + +## Synopsis + +`nix-store` `--restore` *path* + +## Description + +The operation `--restore` unpacks a NAR archive to *path*, which must +not already exist. The archive is read from standard input. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/serve.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/serve.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/serve.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/serve.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,38 @@ +# Name + +`nix-store --serve` - serve local Nix store over SSH + +# Synopsis + +`nix-store` `--serve` [`--write`] + +# Description + +The operation `--serve` provides access to the Nix store over stdin and +stdout, and is intended to be used as a means of providing Nix store +access to a restricted ssh user. + +The following flags are available: + + - `--write`\ + Allow the connected client to request the realization of + derivations. In effect, this can be used to make the host act as a + remote builder. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To turn a host into a build server, the `authorized_keys` file can be +used to provide build access to a given SSH public key: + +```console +$ cat <>/root/.ssh/authorized_keys +command="nice -n20 nix-store --serve --write" ssh-rsa AAAAB3NzaC1yc2EAAAA... +EOF +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/verify.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/verify.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/verify.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/verify.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,36 @@ +# Name + +`nix-store --verify` - check Nix database for consistency + +# Synopsis + +`nix-store` `--verify` [`--check-contents`] [`--repair`] + +# Description + +The operation `--verify` verifies the internal consistency of the Nix +database, and the consistency between the Nix database and the Nix +store. Any inconsistencies encountered are automatically repaired. +Inconsistencies are generally the result of the Nix store or database +being modified by non-Nix tools, or of bugs in Nix itself. + +This operation has the following options: + + - `--check-contents`\ + Checks that the contents of every valid store path has not been + altered by computing a SHA-256 hash of the contents and comparing it + with the hash stored in the Nix database at build time. Paths that + have been modified are printed out. For large stores, + `--check-contents` is obviously quite slow. + + - `--repair`\ + If any valid path is missing from the store, or (if + `--check-contents` is given) the contents of a valid path has been + modified, then try to repair the path by redownloading it. See + `nix-store --repair-path` for details. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store/verify-path.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/verify-path.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store/verify-path.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store/verify-path.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,29 @@ +# Name + +`nix-store --verify-path` - check path contents against Nix database + +## Synopsis + +`nix-store` `--verify-path` *paths…* + +## Description + +The operation `--verify-path` compares the contents of the given store +paths to their cryptographic hashes stored in Nix’s database. For every +changed path, it prints a warning message. The exit status is 0 if no +path has changed, and 1 otherwise. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +## Example + +To verify the integrity of the `svn` command and all its dependencies: + +```console +$ nix-store --verify-path $(nix-store --query --requisites $(which svn)) +``` + diff -Nru nix-2.8.0/doc/manual/src/command-ref/nix-store.md nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store.md --- nix-2.8.0/doc/manual/src/command-ref/nix-store.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/nix-store.md 2023-06-06 10:55:23.000000000 +0000 @@ -13,821 +13,35 @@ The command `nix-store` performs primitive operations on the Nix store. You generally do not need to run this command manually. -`nix-store` takes exactly one *operation* flag which indicates the -subcommand to be performed. These are documented below. +`nix-store` takes exactly one *operation* flag which indicates the subcommand to be performed. The following operations are available: -# Common options +- [`--realise`](./nix-store/realise.md) +- [`--serve`](./nix-store/serve.md) +- [`--gc`](./nix-store/gc.md) +- [`--delete`](./nix-store/delete.md) +- [`--query`](./nix-store/query.md) +- [`--add`](./nix-store/add.md) +- [`--add-fixed`](./nix-store/add-fixed.md) +- [`--verify`](./nix-store/verify.md) +- [`--verify-path`](./nix-store/verify-path.md) +- [`--repair-path`](./nix-store/repair-path.md) +- [`--dump`](./nix-store/dump.md) +- [`--restore`](./nix-store/restore.md) +- [`--export`](./nix-store/export.md) +- [`--import`](./nix-store/import.md) +- [`--optimise`](./nix-store/optimise.md) +- [`--read-log`](./nix-store/read-log.md) +- [`--dump-db`](./nix-store/dump-db.md) +- [`--load-db`](./nix-store/load-db.md) +- [`--print-env`](./nix-store/print-env.md) +- [`--generate-binary-cache-key`](./nix-store/generate-binary-cache-key.md) + +These pages can be viewed offline: -This section lists the options that are common to all operations. These -options are allowed for every subcommand, though they may not always -have an effect. +- `man nix-store-`. - - `--add-root` *path*\ - Causes the result of a realisation (`--realise` and - `--force-realise`) to be registered as a root of the garbage - collector. *path* will be created as a symlink to the resulting - store path. In addition, a uniquely named symlink to *path* will - be created in `/nix/var/nix/gcroots/auto/`. For instance, + Example: `man nix-store-realise` - ```console - $ nix-store --add-root /home/eelco/bla/result -r ... +- `nix-store --help --` - $ ls -l /nix/var/nix/gcroots/auto - lrwxrwxrwx 1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result - - $ ls -l /home/eelco/bla/result - lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r11343n6qd4...-f-spot-0.0.10 - ``` - - Thus, when `/home/eelco/bla/result` is removed, the GC root in the - `auto` directory becomes a dangling symlink and will be ignored by - the collector. - - > **Warning** - > - > Note that it is not possible to move or rename GC roots, since - > the symlink in the `auto` directory will still point to the old - > location. - - If there are multiple results, then multiple symlinks will be - created by sequentially numbering symlinks beyond the first one - (e.g., `foo`, `foo-2`, `foo-3`, and so on). - -# Operation `--realise` - -## Synopsis - -`nix-store` {`--realise` | `-r`} *paths…* [`--dry-run`] - -## Description - -The operation `--realise` essentially “builds” the specified store -paths. Realisation is a somewhat overloaded term: - - - If the store path is a *derivation*, realisation ensures that the - output paths of the derivation are [valid](../glossary.md) (i.e., - the output path and its closure exist in the file system). This - can be done in several ways. First, it is possible that the - outputs are already valid, in which case we are done - immediately. Otherwise, there may be [substitutes](../glossary.md) - that produce the outputs (e.g., by downloading them). Finally, the - outputs can be produced by performing the build action described - by the derivation. - - - If the store path is not a derivation, realisation ensures that the - specified path is valid (i.e., it and its closure exist in the file - system). If the path is already valid, we are done immediately. - Otherwise, the path and any missing paths in its closure may be - produced through substitutes. If there are no (successful) - substitutes, realisation fails. - -The output path of each derivation is printed on standard output. (For -non-derivations argument, the argument itself is printed.) - -The following flags are available: - - - `--dry-run`\ - Print on standard error a description of what packages would be - built or downloaded, without actually performing the operation. - - - `--ignore-unknown`\ - If a non-derivation path does not have a substitute, then silently - ignore it. - - - `--check`\ - This option allows you to check whether a derivation is - deterministic. It rebuilds the specified derivation and checks - whether the result is bitwise-identical with the existing outputs, - printing an error if that’s not the case. The outputs of the - specified derivation must already exist. When used with `-K`, if an - output path is not identical to the corresponding output from the - previous build, the new output path is left in - `/nix/store/name.check.` - - See also the `build-repeat` configuration option, which repeats a - derivation a number of times and prevents its outputs from being - registered as “valid” in the Nix store unless they are identical. - -Special exit codes: - - - `100`\ - Generic build failure, the builder process returned with a non-zero - exit code. - - - `101`\ - Build timeout, the build was aborted because it did not complete - within the specified `timeout`. - - - `102`\ - Hash mismatch, the build output was rejected because it does not - match the [`outputHash` attribute of the - derivation](../expressions/advanced-attributes.md). - - - `104`\ - Not deterministic, the build succeeded in check mode but the - resulting output is not binary reproducible. - -With the `--keep-going` flag it's possible for multiple failures to -occur, in this case the 1xx status codes are or combined using binary -or. - - 1100100 - ^^^^ - |||`- timeout - ||`-- output hash mismatch - |`--- build failure - `---- not deterministic - -## Examples - -This operation is typically used to build store derivations produced by -[`nix-instantiate`](nix-instantiate.md): - -```console -$ nix-store -r $(nix-instantiate ./test.nix) -/nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1 -``` - -This is essentially what [`nix-build`](nix-build.md) does. - -To test whether a previously-built derivation is deterministic: - -```console -$ nix-build '' -A hello --check -K -``` - -# Operation `--serve` - -## Synopsis - -`nix-store` `--serve` [`--write`] - -## Description - -The operation `--serve` provides access to the Nix store over stdin and -stdout, and is intended to be used as a means of providing Nix store -access to a restricted ssh user. - -The following flags are available: - - - `--write`\ - Allow the connected client to request the realization of - derivations. In effect, this can be used to make the host act as a - remote builder. - -## Examples - -To turn a host into a build server, the `authorized_keys` file can be -used to provide build access to a given SSH public key: - -```console -$ cat <>/root/.ssh/authorized_keys -command="nice -n20 nix-store --serve --write" ssh-rsa AAAAB3NzaC1yc2EAAAA... -EOF -``` - -# Operation `--gc` - -## Synopsis - -`nix-store` `--gc` [`--print-roots` | `--print-live` | `--print-dead`] [`--max-freed` *bytes*] - -## Description - -Without additional flags, the operation `--gc` performs a garbage -collection on the Nix store. That is, all paths in the Nix store not -reachable via file system references from a set of “roots”, are deleted. - -The following suboperations may be specified: - - - `--print-roots`\ - This operation prints on standard output the set of roots used by - the garbage collector. - - - `--print-live`\ - This operation prints on standard output the set of “live” store - paths, which are all the store paths reachable from the roots. Live - paths should never be deleted, since that would break consistency — - it would become possible that applications are installed that - reference things that are no longer present in the store. - - - `--print-dead`\ - This operation prints out on standard output the set of “dead” store - paths, which is just the opposite of the set of live paths: any path - in the store that is not live (with respect to the roots) is dead. - -By default, all unreachable paths are deleted. The following options -control what gets deleted and in what order: - - - `--max-freed` *bytes*\ - Keep deleting paths until at least *bytes* bytes have been deleted, - then stop. The argument *bytes* can be followed by the - multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB - or TiB units. - -The behaviour of the collector is also influenced by the -`keep-outputs` and `keep-derivations` settings in the Nix -configuration file. - -By default, the collector prints the total number of freed bytes when it -finishes (or when it is interrupted). With `--print-dead`, it prints the -number of bytes that would be freed. - -## Examples - -To delete all unreachable paths, just do: - -```console -$ nix-store --gc -deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv' -... -8825586 bytes freed (8.42 MiB) -``` - -To delete at least 100 MiBs of unreachable paths: - -```console -$ nix-store --gc --max-freed $((100 * 1024 * 1024)) -``` - -# Operation `--delete` - -## Synopsis - -`nix-store` `--delete` [`--ignore-liveness`] *paths…* - -## Description - -The operation `--delete` deletes the store paths *paths* from the Nix -store, but only if it is safe to do so; that is, when the path is not -reachable from a root of the garbage collector. This means that you can -only delete paths that would also be deleted by `nix-store --gc`. Thus, -`--delete` is a more targeted version of `--gc`. - -With the option `--ignore-liveness`, reachability from the roots is -ignored. However, the path still won’t be deleted if there are other -paths in the store that refer to it (i.e., depend on it). - -## Example - -```console -$ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4 -0 bytes freed (0.00 MiB) -error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive -``` - -# Operation `--query` - -## Synopsis - -`nix-store` {`--query` | `-q`} - {`--outputs` | `--requisites` | `-R` | `--references` | - `--referrers` | `--referrers-closure` | `--deriver` | `-d` | - `--graph` | `--tree` | `--binding` *name* | `-b` *name* | `--hash` | - `--size` | `--roots`} - [`--use-output`] [`-u`] [`--force-realise`] [`-f`] - *paths…* - -## Description - -The operation `--query` displays various bits of information about the -store paths . The queries are described below. At most one query can be -specified. The default query is `--outputs`. - -The paths *paths* may also be symlinks from outside of the Nix store, to -the Nix store. In that case, the query is applied to the target of the -symlink. - -## Common query options - - - `--use-output`; `-u`\ - For each argument to the query that is a store derivation, apply the - query to the output path of the derivation instead. - - - `--force-realise`; `-f`\ - Realise each argument to the query first (see [`nix-store - --realise`](#operation---realise)). - -## Queries - - - `--outputs`\ - Prints out the [output paths](../glossary.md) of the store - derivations *paths*. These are the paths that will be produced when - the derivation is built. - - - `--requisites`; `-R`\ - Prints out the [closure](../glossary.md) of the store path *paths*. - - This query has one option: - - - `--include-outputs` - Also include the existing output paths of store derivations, - and their closures. - - This query can be used to implement various kinds of deployment. A - *source deployment* is obtained by distributing the closure of a - store derivation. A *binary deployment* is obtained by distributing - the closure of an output path. A *cache deployment* (combined - source/binary deployment, including binaries of build-time-only - dependencies) is obtained by distributing the closure of a store - derivation and specifying the option `--include-outputs`. - - - `--references`\ - Prints the set of [references](../glossary.md) of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) - - - `--referrers`\ - Prints the set of *referrers* of the store paths *paths*, that is, - the store paths currently existing in the Nix store that refer to - one of *paths*. Note that contrary to the references, the set of - referrers is not constant; it can change as store paths are added or - removed. - - - `--referrers-closure`\ - Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or - indirectly refer to one of *paths*. These are all the path currently - in the Nix store that are dependent on *paths*. - - - `--deriver`; `-d`\ - Prints the [deriver](../glossary.md) of the store paths *paths*. If - the path has no deriver (e.g., if it is a source file), or if the - deriver is not known (e.g., in the case of a binary-only - deployment), the string `unknown-deriver` is printed. - - - `--graph`\ - Prints the references graph of the store paths *paths* in the format - of the `dot` tool of AT\&T's [Graphviz - package](http://www.graphviz.org/). This can be used to visualise - dependency graphs. To obtain a build-time dependency graph, apply - this to a store derivation. To obtain a runtime dependency graph, - apply it to an output path. - - - `--tree`\ - Prints the references graph of the store paths *paths* as a nested - ASCII tree. References are ordered by descending closure size; this - tends to flatten the tree, making it more readable. The query only - recurses into a store path when it is first encountered; this - prevents a blowup of the tree representation of the graph. - - - `--graphml`\ - Prints the references graph of the store paths *paths* in the - [GraphML](http://graphml.graphdrawing.org/) file format. This can be - used to visualise dependency graphs. To obtain a build-time - dependency graph, apply this to a store derivation. To obtain a - runtime dependency graph, apply it to an output path. - - - `--binding` *name*; `-b` *name*\ - Prints the value of the attribute *name* (i.e., environment - variable) of the store derivations *paths*. It is an error for a - derivation to not have the specified attribute. - - - `--hash`\ - Prints the SHA-256 hash of the contents of the store paths *paths* - (that is, the hash of the output of `nix-store --dump` on the given - paths). Since the hash is stored in the Nix database, this is a fast - operation. - - - `--size`\ - Prints the size in bytes of the contents of the store paths *paths* - — to be precise, the size of the output of `nix-store --dump` on - the given paths. Note that the actual disk space required by the - store paths may be higher, especially on filesystems with large - cluster sizes. - - - `--roots`\ - Prints the garbage collector roots that point, directly or - indirectly, at the store paths *paths*. - -## Examples - -Print the closure (runtime dependencies) of the `svn` program in the -current user environment: - -```console -$ nix-store -qR $(which svn) -/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 -/nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4 -... -``` - -Print the build-time dependencies of `svn`: - -```console -$ nix-store -qR $(nix-store -qd $(which svn)) -/nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv -/nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh -/nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv -... lots of other paths ... -``` - -The difference with the previous example is that we ask the closure of -the derivation (`-qd`), not the closure of the output path that contains -`svn`. - -Show the build-time dependencies as a tree: - -```console -$ nix-store -q --tree $(nix-store -qd $(which svn)) -/nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv -+---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh -+---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv -| +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash -| +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh -... -``` - -Show all paths that depend on the same OpenSSL library as `svn`: - -```console -$ nix-store -q --referrers $(nix-store -q --binding openssl $(nix-store -qd $(which svn))) -/nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0 -/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 -/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3 -/nix/store/l51240xqsgg8a7yrbqdx1rfzyv6l26fx-lynx-2.8.5 -``` - -Show all paths that directly or indirectly depend on the Glibc (C -library) used by `svn`: - -```console -$ nix-store -q --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}') -/nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2 -/nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4 -... -``` - -Note that `ldd` is a command that prints out the dynamic libraries used -by an ELF executable. - -Make a picture of the runtime dependency graph of the current user -environment: - -```console -$ nix-store -q --graph ~/.nix-profile | dot -Tps > graph.ps -$ gv graph.ps -``` - -Show every garbage collector root that points to a store path that -depends on `svn`: - -```console -$ nix-store -q --roots $(which svn) -/nix/var/nix/profiles/default-81-link -/nix/var/nix/profiles/default-82-link -/nix/var/nix/profiles/per-user/eelco/profile-97-link -``` - -# Operation `--add` - -## Synopsis - -`nix-store` `--add` *paths…* - -## Description - -The operation `--add` adds the specified paths to the Nix store. It -prints the resulting paths in the Nix store on standard output. - -## Example - -```console -$ nix-store --add ./foo.c -/nix/store/m7lrha58ph6rcnv109yzx1nk1cj7k7zf-foo.c -``` - -# Operation `--add-fixed` - -## Synopsis - -`nix-store` `--add-fixed` [`--recursive`] *algorithm* *paths…* - -## Description - -The operation `--add-fixed` adds the specified paths to the Nix store. -Unlike `--add` paths are registered using the specified hashing -algorithm, resulting in the same output path as a fixed-output -derivation. This can be used for sources that are not available from a -public url or broke since the download expression was written. - -This operation has the following options: - - - `--recursive`\ - Use recursive instead of flat hashing mode, used when adding - directories to the store. - -## Example - -```console -$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz -/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz -``` - -# Operation `--verify` - -## Synopsis - -`nix-store` `--verify` [`--check-contents`] [`--repair`] - -## Description - -The operation `--verify` verifies the internal consistency of the Nix -database, and the consistency between the Nix database and the Nix -store. Any inconsistencies encountered are automatically repaired. -Inconsistencies are generally the result of the Nix store or database -being modified by non-Nix tools, or of bugs in Nix itself. - -This operation has the following options: - - - `--check-contents`\ - Checks that the contents of every valid store path has not been - altered by computing a SHA-256 hash of the contents and comparing it - with the hash stored in the Nix database at build time. Paths that - have been modified are printed out. For large stores, - `--check-contents` is obviously quite slow. - - - `--repair`\ - If any valid path is missing from the store, or (if - `--check-contents` is given) the contents of a valid path has been - modified, then try to repair the path by redownloading it. See - `nix-store --repair-path` for details. - -# Operation `--verify-path` - -## Synopsis - -`nix-store` `--verify-path` *paths…* - -## Description - -The operation `--verify-path` compares the contents of the given store -paths to their cryptographic hashes stored in Nix’s database. For every -changed path, it prints a warning message. The exit status is 0 if no -path has changed, and 1 otherwise. - -## Example - -To verify the integrity of the `svn` command and all its dependencies: - -```console -$ nix-store --verify-path $(nix-store -qR $(which svn)) -``` - -# Operation `--repair-path` - -## Synopsis - -`nix-store` `--repair-path` *paths…* - -## Description - -The operation `--repair-path` attempts to “repair” the specified paths -by redownloading them using the available substituters. If no -substitutes are available, then repair is not possible. - -> **Warning** -> -> During repair, there is a very small time window during which the old -> path (if it exists) is moved out of the way and replaced with the new -> path. If repair is interrupted in between, then the system may be left -> in a broken state (e.g., if the path contains a critical system -> component like the GNU C Library). - -## Example - -```console -$ nix-store --verify-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 -path `/nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13' was modified! - expected hash `2db57715ae90b7e31ff1f2ecb8c12ec1cc43da920efcbe3b22763f36a1861588', - got `481c5aa5483ebc97c20457bb8bca24deea56550d3985cda0027f67fe54b808e4' - -$ nix-store --repair-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 -fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'... -… -``` - -# Operation `--dump` - -## Synopsis - -`nix-store` `--dump` *path* - -## Description - -The operation `--dump` produces a NAR (Nix ARchive) file containing the -contents of the file system tree rooted at *path*. The archive is -written to standard output. - -A NAR archive is like a TAR or Zip archive, but it contains only the -information that Nix considers important. For instance, timestamps are -elided because all files in the Nix store have their timestamp set to 0 -anyway. Likewise, all permissions are left out except for the execute -bit, because all files in the Nix store have 444 or 555 permission. - -Also, a NAR archive is *canonical*, meaning that “equal” paths always -produce the same NAR archive. For instance, directory entries are -always sorted so that the actual on-disk order doesn’t influence the -result. This means that the cryptographic hash of a NAR dump of a -path is usable as a fingerprint of the contents of the path. Indeed, -the hashes of store paths stored in Nix’s database (see `nix-store -q ---hash`) are SHA-256 hashes of the NAR dump of each store path. - -NAR archives support filenames of unlimited length and 64-bit file -sizes. They can contain regular files, directories, and symbolic links, -but not other types of files (such as device nodes). - -A Nix archive can be unpacked using `nix-store ---restore`. - -# Operation `--restore` - -## Synopsis - -`nix-store` `--restore` *path* - -## Description - -The operation `--restore` unpacks a NAR archive to *path*, which must -not already exist. The archive is read from standard input. - -# Operation `--export` - -## Synopsis - -`nix-store` `--export` *paths…* - -## Description - -The operation `--export` writes a serialisation of the specified store -paths to standard output in a format that can be imported into another -Nix store with `nix-store --import`. This is like `nix-store ---dump`, except that the NAR archive produced by that command doesn’t -contain the necessary meta-information to allow it to be imported into -another Nix store (namely, the set of references of the path). - -This command does not produce a *closure* of the specified paths, so if -a store path references other store paths that are missing in the target -Nix store, the import will fail. To copy a whole closure, do something -like: - -```console -$ nix-store --export $(nix-store -qR paths) > out -``` - -To import the whole closure again, run: - -```console -$ nix-store --import < out -``` - -# Operation `--import` - -## Synopsis - -`nix-store` `--import` - -## Description - -The operation `--import` reads a serialisation of a set of store paths -produced by `nix-store --export` from standard input and adds those -store paths to the Nix store. Paths that already exist in the Nix store -are ignored. If a path refers to another path that doesn’t exist in the -Nix store, the import fails. - -# Operation `--optimise` - -## Synopsis - -`nix-store` `--optimise` - -## Description - -The operation `--optimise` reduces Nix store disk space usage by finding -identical files in the store and hard-linking them to each other. It -typically reduces the size of the store by something like 25-35%. Only -regular files and symlinks are hard-linked in this manner. Files are -considered identical when they have the same NAR archive serialisation: -that is, regular files must have the same contents and permission -(executable or non-executable), and symlinks must have the same -contents. - -After completion, or when the command is interrupted, a report on the -achieved savings is printed on standard error. - -Use `-vv` or `-vvv` to get some progress indication. - -## Example - -```console -$ nix-store --optimise -hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' -... -541838819 bytes (516.74 MiB) freed by hard-linking 54143 files; -there are 114486 files with equal contents out of 215894 files in total -``` - -# Operation `--read-log` - -## Synopsis - -`nix-store` {`--read-log` | `-l`} *paths…* - -## Description - -The operation `--read-log` prints the build log of the specified store -paths on standard output. The build log is whatever the builder of a -derivation wrote to standard output and standard error. If a store path -is not a derivation, the deriver of the store path is used. - -Build logs are kept in `/nix/var/log/nix/drvs`. However, there is no -guarantee that a build log is available for any particular store path. -For instance, if the path was downloaded as a pre-built binary through a -substitute, then the log is unavailable. - -## Example - -```console -$ nix-store -l $(which ktorrent) -building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1 -unpacking sources -unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz -ktorrent-2.2.1/ -ktorrent-2.2.1/NEWS -... -``` - -# Operation `--dump-db` - -## Synopsis - -`nix-store` `--dump-db` [*paths…*] - -## Description - -The operation `--dump-db` writes a dump of the Nix database to standard -output. It can be loaded into an empty Nix store using `--load-db`. This -is useful for making backups and when migrating to different database -schemas. - -By default, `--dump-db` will dump the entire Nix database. When one or -more store paths is passed, only the subset of the Nix database for -those store paths is dumped. As with `--export`, the user is responsible -for passing all the store paths for a closure. See `--export` for an -example. - -# Operation `--load-db` - -## Synopsis - -`nix-store` `--load-db` - -## Description - -The operation `--load-db` reads a dump of the Nix database created by -`--dump-db` from standard input and loads it into the Nix database. - -# Operation `--print-env` - -## Synopsis - -`nix-store` `--print-env` *drvpath* - -## Description - -The operation `--print-env` prints out the environment of a derivation -in a format that can be evaluated by a shell. The command line arguments -of the builder are placed in the variable `_args`. - -## Example - -```console -$ nix-store --print-env $(nix-instantiate '' -A firefox) -… -export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2' -export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv' -export system; system='x86_64-linux' -export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-builder.sh' -``` - -# Operation `--generate-binary-cache-key` - -## Synopsis - -`nix-store` `--generate-binary-cache-key` *key-name* *secret-key-file* *public-key-file* - -## Description - -This command generates an [Ed25519 key pair](http://ed25519.cr.yp.to/) -that can be used to create a signed binary cache. It takes three -mandatory parameters: - -1. A key name, such as `cache.example.org-1`, that is used to look up - keys on the client when it verifies signatures. It can be anything, - but it’s suggested to use the host name of your cache (e.g. - `cache.example.org`) with a suffix denoting the number of the key - (to be incremented every time you need to revoke a key). - -2. The file name where the secret key is to be stored. - -3. The file name where the public key is to be stored. + Example: `nix-store --help --realise` diff -Nru nix-2.8.0/doc/manual/src/command-ref/opt-common.md nix-2.16.1+dfsg/doc/manual/src/command-ref/opt-common.md --- nix-2.8.0/doc/manual/src/command-ref/opt-common.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/command-ref/opt-common.md 2023-06-06 10:55:23.000000000 +0000 @@ -2,13 +2,13 @@ Most Nix commands accept the following command-line options: - - `--help`\ + - [`--help`](#opt-help)\ Prints out a summary of the command syntax and exits. - - `--version`\ + - [`--version`](#opt-version)\ Prints out the Nix version number on standard output and exits. - - `--verbose` / `-v`\ + - [`--verbose`](#opt-verbose) / `-v`\ Increases the level of verbosity of diagnostic messages printed on standard error. For each Nix operation, the information printed on standard output is well-defined; any diagnostic information is @@ -37,14 +37,14 @@ - 5\ “Vomit”: print vast amounts of debug information. - - `--quiet`\ + - [`--quiet`](#opt-quiet)\ Decreases the level of verbosity of diagnostic messages printed on standard error. This is the inverse option to `-v` / `--verbose`. This option may be specified repeatedly. See the previous verbosity levels list. - - `--log-format` *format*\ + - [`--log-format`](#opt-log-format) *format*\ This option can be used to change the output of the log format, with *format* being one of: @@ -66,14 +66,14 @@ - bar-with-logs\ Display the raw logs, with the progress bar at the bottom. - - `--no-build-output` / `-Q`\ + - [`--no-build-output`](#opt-no-build-output) / `-Q`\ By default, output written by builders to standard output and standard error is echoed to the Nix command's standard error. This option suppresses this behaviour. Note that the builder's standard output and error are always written to a log file in `prefix/nix/var/log/nix`. - - `--max-jobs` / `-j` *number*\ + - [`--max-jobs`](#opt-max-jobs) / `-j` *number*\ Sets the maximum number of build jobs that Nix will perform in parallel to the specified number. Specify `auto` to use the number of CPUs in the system. The default is specified by the `max-jobs` @@ -83,7 +83,7 @@ Setting it to `0` disallows building on the local machine, which is useful when you want builds to happen only on remote builders. - - `--cores`\ + - [`--cores`](#opt-cores)\ Sets the value of the `NIX_BUILD_CORES` environment variable in the invocation of builders. Builders can use this variable at their discretion to control the maximum amount of parallelism. For @@ -94,18 +94,18 @@ means that the builder should use all available CPU cores in the system. - - `--max-silent-time`\ + - [`--max-silent-time`](#opt-max-silent-time)\ Sets the maximum number of seconds that a builder can go without producing any data on standard output or standard error. The default is specified by the `max-silent-time` configuration setting. `0` means no time-out. - - `--timeout`\ + - [`--timeout`](#opt-timeout)\ Sets the maximum number of seconds that a builder can run. The default is specified by the `timeout` configuration setting. `0` means no timeout. - - `--keep-going` / `-k`\ + - [`--keep-going`](#opt-keep-going) / `-k`\ Keep going in case of failed builds, to the greatest extent possible. That is, if building an input of some derivation fails, Nix will still build the other inputs, but not the derivation @@ -113,13 +113,13 @@ for builds of substitutes), possibly killing builds in progress (in case of parallel or distributed builds). - - `--keep-failed` / `-K`\ + - [`--keep-failed`](#opt-keep-failed) / `-K`\ Specifies that in case of a build failure, the temporary directory (usually in `/tmp`) in which the build takes place should not be deleted. The path of the build directory is printed as an informational message. - - `--fallback`\ + - [`--fallback`](#opt-fallback)\ Whenever Nix attempts to build a derivation for which substitutes are known for each output path, but realising the output paths through the substitutes fails, fall back on building the derivation. @@ -134,18 +134,18 @@ failure in obtaining the substitutes to lead to a full build from source (with the related consumption of resources). - - `--readonly-mode`\ + - [`--readonly-mode`](#opt-readonly-mode)\ When this option is used, no attempt is made to open the Nix database. Most Nix operations do need database access, so those operations will fail. - - `--arg` *name* *value*\ + - [`--arg`](#opt-arg) *name* *value*\ This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. It can automatically call functions for which every argument has a [default - value](../expressions/language-constructs.md#functions) (e.g., + value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a @@ -162,28 +162,28 @@ }: ... ``` - So if you call this Nix expression (e.g., when you do `nix-env -iA + So if you call this Nix expression (e.g., when you do `nix-env --install --attr pkgname`), the function will be called automatically using the - value [`builtins.currentSystem`](../expressions/builtins.md) for + value [`builtins.currentSystem`](@docroot@/language/builtins.md) for the `system` argument. You can override this using `--arg`, e.g., - `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that + `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`. (Note that since the argument is a Nix string literal, you have to escape the quotes.) - - `--argstr` *name* *value*\ + - [`--argstr`](#opt-argstr) *name* *value*\ This option is like `--arg`, only the value is not a Nix expression but a string. So instead of `--arg system \"i686-linux\"` (the outer quotes are to keep the shell happy) you can say `--argstr system i686-linux`. - - `--attr` / `-A` *attrPath*\ + - [`--attr`](#opt-attr) / `-A` *attrPath*\ Select an attribute from the top-level Nix expression being evaluated. (`nix-env`, `nix-instantiate`, `nix-build` and `nix-shell` only.) The *attribute path* *attrPath* is a sequence of attribute names separated by dots. For instance, given a top-level Nix expression *e*, the attribute path `xorg.xorgserver` would cause the expression `e.xorg.xorgserver` to be used. See - [`nix-env --install`](nix-env.md#operation---install) for some + [`nix-env --install`](@docroot@/command-ref/nix-env/install.md) for some concrete examples. In addition to attribute names, you can also specify array indices. @@ -191,7 +191,7 @@ attribute of the fourth element of the array in the `foo` attribute of the top-level expression. - - `--expr` / `-E`\ + - [`--expr`](#opt-expr) / `-E`\ Interpret the command line arguments as a list of Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions. (`nix-instantiate`, `nix-build` and `nix-shell` only.) @@ -199,20 +199,19 @@ For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for - use, give your expression to the `nix-shell -p` convenience flag + use, give your expression to the `nix-shell --packages ` convenience flag instead. - - `-I` *path*\ - Add a path to the Nix expression search path. This option may be - given multiple times. See the `NIX_PATH` environment variable for - information on the semantics of the Nix search path. Paths added - through `-I` take precedence over `NIX_PATH`. + - [`-I`](#opt-I) *path*\ + Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + This option may be given multiple times. + Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). - - `--option` *name* *value*\ + - [`--option`](#opt-option) *name* *value*\ Set the Nix configuration option *name* to *value*. This overrides settings in the Nix configuration file (see nix.conf5). - - `--repair`\ + - [`--repair`](#opt-repair)\ Fix corrupted or missing store paths by redownloading or rebuilding them. Note that this is slow because it requires computing a cryptographic hash of the contents of every path in the closure of diff -Nru nix-2.8.0/doc/manual/src/contributing/cli-guideline.md nix-2.16.1+dfsg/doc/manual/src/contributing/cli-guideline.md --- nix-2.8.0/doc/manual/src/contributing/cli-guideline.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/contributing/cli-guideline.md 2023-06-06 10:55:23.000000000 +0000 @@ -389,6 +389,88 @@ happen when TTY is not detected on STDERR. We should not display progress / status section, but only print warnings and errors. +## Returning future proof JSON + +The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this. + +Two definitions are helpful here, because while JSON only defines one "key-value" +object type, we use it to cover two use cases: + + - **dictionary**: a map from names to value that all have the same type. In + C++ this would be a `std::map` with string keys. + - **record**: a fixed set of attributes each with their own type. In C++, this + would be represented by a `struct`. + +It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type. + +This leads to the following guidelines: + + - The top-level (root) value must be a record. + + Otherwise, one can not change the structure of a command's output. + + - The value of a dictionary item must be a record. + + Otherwise, the item type can not be extended. + + - List items should be records. + + Otherwise, one can not change the structure of the list items. + + If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records. + + - Streaming JSON should return records. + + An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records. + +### Examples + + +This is bad, because all keys must be assumed to be store implementations: + +```json +{ + "local": { ... }, + "remote": { ... }, + "http": { ... } +} +``` + +This is good, because the it is extensible at the root, and is somewhat self-documenting: + +```json +{ + "storeTypes": { "local": { ... }, ... }, + "pluginSupport": true +} +``` + +While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. +For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. + + + +The following representation is bad because it is not extensible: + +```json +{ "outputs": [ "out" "bin" ] } +``` + +However, simply converting everything to records is not enough, because the order of outputs must be preserved: + +```json +{ "outputs": { "bin": {}, "out": {} } } +``` + +The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. +While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. + +This representation is extensible and preserves the ordering: + +```json +{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } +``` + ## Dialog with the user CLIs don't always make it clear when an action has taken place. For every diff -Nru nix-2.8.0/doc/manual/src/contributing/experimental-features.md nix-2.16.1+dfsg/doc/manual/src/contributing/experimental-features.md --- nix-2.8.0/doc/manual/src/contributing/experimental-features.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/contributing/experimental-features.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,95 @@ +This section describes the notion of *experimental features*, and how it fits into the big picture of the development of Nix. + +# What are experimental features? + +Experimental features are considered unstable, which means that they can be changed or removed at any time. +Users must explicitly enable them by toggling the associated [experimental feature flags](@docroot@/command-ref/conf-file.md#conf-experimental-features). +This allows accessing unstable functionality without unwittingly relying on it. + +Experimental feature flags were first introduced in [Nix 2.4](@docroot@/release-notes/rl-2.4.md). +Before that, Nix did have experimental features, but they were not guarded by flags and were merely documented as unstable. +This was a source of confusion and controversy. + +# When should a new feature be marked experimental? + +A change in the Nix codebase should be guarded by an experimental feature flag if it is considered likely to be reverted or adapted in a backwards-incompatible manner after gathering more experience with it in practice. + +Examples: + +- Changes to the Nix language, such as new built-ins, syntactic or semantic changes, etc. +- Changes to the command-line interface + +# Lifecycle of an experimental feature + +Experimental features have to be treated on a case-by-case basis. +However, the standard workflow for an experimental feature is as follows: + +- A new feature is implemented in a *pull request* + - It is guarded by an experimental feature flag that is disabled by default +- The pull request is merged, the *experimental* feature ends up in a release + - Using the feature requires explicitly enabling it, signifying awareness of the potential risks + - Being experimental, the feature can still be changed arbitrarily +- The feature can be *removed* + - The associated experimental feature flag is also removed +- The feature can be declared *stable* + - The associated experimental feature flag is removed + - There should be enough evidence of users having tried the feature, such as feedback, fixed bugs, demonstrations of how it is put to use + - Maintainers must feel confident that: + - The feature is designed and implemented sensibly, that it is fit for purpose + - Potential interactions are well-understood + - Stabilising the feature will not incur an outsized maintenance burden in the future + +The following diagram illustrates the process: + +``` + .------. + | idea | + '------' + | + discussion, design, implementation + | + | .-------. + | | | + v v | + .--------------. review + | pull request | | + '--------------' | + | ^ | | + | | '-------' + .---' '----. + | | + merge user feedback, + | (breaking) changes + | | + '---. .----' + | | + v | + +--------------+ + .---| experimental |----. + | +--------------+ | + | | +decision to stabilise decision against + | keeping the feature + | | + v v + +--------+ +---------+ + | stable | | removed | + +--------+ +---------+ +``` + +# Relation to the RFC process + +Experimental features and [RFCs](https://github.com/NixOS/rfcs/) both allow approaching substantial changes while minimizing the risk. +However they serve different purposes: + +- An experimental feature enables developers to iterate on and deliver a new idea without committing to it or requiring a costly long-running fork. + It is primarily an issue of *implementation*, targeting Nix developers and early testers. +- The goal of an RFC is to make explicit all the implications of a change: + Explain why it is wanted, which new use-cases it enables, which interface changes it requires, etc. + It is primarily an issue of *design* and *communication*, targeting the broader community. + +This means that experimental features and RFCs are orthogonal mechanisms, and can be used independently or together as needed. + +# Currently available experimental features + +{{#include ./experimental-feature-descriptions.md}} diff -Nru nix-2.8.0/doc/manual/src/contributing/hacking.md nix-2.16.1+dfsg/doc/manual/src/contributing/hacking.md --- nix-2.8.0/doc/manual/src/contributing/hacking.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/contributing/hacking.md 2023-06-06 10:55:23.000000000 +0000 @@ -8,52 +8,83 @@ $ cd nix ``` -To build Nix for the current operating system/architecture use +The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. + +[installation instructions]: ../installation/installation.md + +## Nix with flakes + +This section assumes you are using Nix with [flakes] enabled. See the [next section](#classic-nix) for equivalent instructions which don't require flakes. + +[flakes]: ../command-ref/new-cli/nix3-flake.md#description + +To build all dependencies and start a shell in which all environment +variables are set up so that those dependencies can be found: ```console -$ nix-build +$ nix develop ``` -or if you have a flake-enabled nix: +This shell also adds `./outputs/bin/nix` to your `$PATH` so you can run `nix` immediately after building it. + +To get a shell with one of the other [supported compilation environments](#compilation-environments): ```console -$ nix build +$ nix develop .#native-clang11StdenvPackages ``` -This will build `defaultPackage` attribute defined in the `flake.nix` -file. To build for other platforms add one of the following suffixes to -it: aarch64-linux, i686-linux, x86\_64-darwin, x86\_64-linux. i.e. +> **Note** +> +> Use `ccacheStdenv` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: ```console -$ nix-build -A defaultPackage.x86_64-linux +[nix-shell]$ ./bootstrap.sh +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES ``` -To build all dependencies and start a shell in which all environment -variables are set up so that those dependencies can be found: +To install it in `$(pwd)/outputs` and test it: ```console -$ nix-shell +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ nix --version +nix (Nix) 2.12 ``` -or if you have a flake-enabled nix: +To build a release version of Nix: ```console -$ nix develop +$ nix build ``` -To get a shell with a different compilation environment (e.g. stdenv, -gccStdenv, clangStdenv, clang11Stdenv): +You can also build Nix for one of the [supported target platforms](#target-platforms). + +## Classic Nix + +This section is for Nix without [flakes]. + +To build all dependencies and start a shell in which all environment +variables are set up so that those dependencies can be found: ```console -$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages +$ nix-shell ``` -or if you have a flake-enabled nix: +To get a shell with one of the other [supported compilation environments](#compilation-environments): ```console -$ nix develop .#clang11StdenvPackages +$ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages ``` +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + To build Nix itself in this shell: ```console @@ -68,29 +99,325 @@ [nix-shell]$ make install [nix-shell]$ make installcheck -j $NIX_BUILD_CORES [nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 3.0 +nix (Nix) 2.12 +``` + +To build Nix for the current operating system and CPU architecture use + +```console +$ nix-build ``` -To run a functional test: +You can also build Nix for one of the [supported target platforms](#target-platforms). + +## Platforms + +As specified in [`flake.nix`], Nix can be built for various platforms: + +- `aarch64-linux` +- `i686-linux` +- `x86_64-darwin` +- `x86_64-linux` + +[`flake.nix`]: https://github.com/nixos/nix/blob/master/flake.nix + +In order to build Nix for a different platform than the one you're currently +on, you need to have some way for your system Nix to build code for that +platform. Common solutions include [remote builders] and [binfmt emulation] +(only supported on NixOS). + +[remote builders]: ../advanced-topics/distributed-builds.md +[binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems + +These solutions let Nix perform builds as if you're on the native platform, so +executing the build is as simple as ```console -make tests/test-name-should-auto-complete.sh.test +$ nix build .#packages.aarch64-linux.default ``` -To run the unit-tests for C++ code: +for flake-enabled Nix, or +```console +$ nix-build --attr packages.aarch64-linux.default ``` -make check + +for classic Nix. + +You can use any of the other supported platforms in place of `aarch64-linux`. + +Cross-compiled builds are available for ARMv6 and ARMv7, and Nix on unsupported platforms can be bootstrapped by adding more `crossSystems` in `flake.nix`. + +## Compilation environments + +Nix can be compiled using multiple environments: + +- `stdenv`: default; +- `gccStdenv`: force the use of `gcc` compiler; +- `clangStdenv`: force the use of `clang` compiler; +- `ccacheStdenv`: enable [ccache], a compiler cache to speed up compilation. + +To build with one of those environments, you can use + +```console +$ nix build .#nix-ccacheStdenv ``` -If you have a flakes-enabled Nix you can replace: +for flake-enabled Nix, or ```console -$ nix-shell +$ nix-build --attr nix-ccacheStdenv ``` -by: +for classic Nix. + +You can use any of the other supported environments in place of `nix-ccacheStdenv`. + +## Editor integration + +The `clangd` LSP server is installed by default on the `clang`-based `devShell`s. +See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#nix-with-flakes) or in [classic Nix](#classic-nix). + +To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -$ nix develop +make clean && bear -- make -j$NIX_BUILD_CORES install +``` + +Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). + +> **Note** +> +> For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`. +> Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). +> Editor-specific setup is typically opinionated, so we will not cover it here in more detail. + +## Running tests + +### Unit-tests + +The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined +under `src/{library_name}/tests` using the +[googletest](https://google.github.io/googletest/) and +[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. + +You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. + +### Functional tests + +The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. +Each test is a bash script. + +The whole test suite can be run with: + +```shell-session +$ make install && make installcheck +ran test tests/foo.sh... [PASS] +ran test tests/bar.sh... [PASS] +... +``` + +Individual tests can be run with `make`: + +```shell-session +$ make tests/${testName}.sh.test +ran test tests/${testName}.sh... [PASS] +``` + +or without `make`: + +```shell-session +$ ./mk/run-test.sh tests/${testName}.sh +ran test tests/${testName}.sh... [PASS] +``` + +To see the complete output, one can also run: + +```shell-session +$ ./mk/debug-test.sh tests/${testName}.sh ++ foo +output from foo ++ bar +output from bar +... +``` + +The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. + +#### Debugging failing functional tests + +When a functional test fails, it usually does so somewhere in the middle of the script. + +To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB. + +For example, if the script looks like: + +```bash +foo +nix blah blub +bar +``` +edit it like so: + +```diff + foo +-nix blah blub ++gdb --args nix blah blub + bar +``` + +Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: + +```shell-session +$ ./mk/debug-test.sh tests/${testName}.sh +... ++ gdb blash blub +GNU gdb (GDB) 12.1 +... +(gdb) +``` + +One can debug the Nix invocation in all the usual ways. +For example, enter `run` to start the Nix invocation. + +### Integration tests + +The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. +These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. +Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). + +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` + +### Installer tests + +After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. + +Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): + +- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: + - `x86_64-linux` + - `armv6l-linux` + - `armv7l-linux` + - `x86_64-darwin` + +- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. + +#### One-time setup + +1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). +2. At cachix.org: + - Create or log in to an account. + - Create a Cachix cache using the format `-nix-install-tests`. + - Navigate to the new cache > Settings > Auth Tokens. + - Generate a new Cachix auth token and copy the generated value. +3. At github.com: + - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. + - Name the secret `CACHIX_AUTH_TOKEN`. + - Paste the copied value of the Cachix cache auth token. + +#### Using the CI-generated installer for manual testing + +After the CI run completes, you can check the output to extract the installer URL: +1. Click into the detailed view of the CI run. +2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). +3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) +4. Copy the value of `install_url` +5. To generate an install command, plug this `install_url` and your GitHub username into this template: + + ```console + curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve + ``` + + + +### Checking links in the manual + +The build checks for broken internal links. +This happens late in the process, so `nix build` is not suitable for iterating. +To build the manual incrementally, run: + +```console +make html -j $NIX_BUILD_CORES +``` + +In order to reflect changes to the [Makefile], clear all generated files before re-building: + +[Makefile]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk + +```console +rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES +``` + +[`mdbook-linkcheck`] does not implement checking [URI fragments] yet. + +[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck +[URI fragments]: https://en.m.wikipedia.org/wiki/URI_fragment + +#### `@docroot@` variable + +`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. + +If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. + +If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. +See existing `@docroot@` logic in the [Makefile]. +Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`. + +## API documentation + +Doxygen API documentation is [available +online](https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs). You +can also build and view it yourself: + +```console +# nix build .#hydraJobs.internal-api-docs +# xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside a `nix develop` shell by running: + +``` +# make internal-api-html +# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html ``` + +## Coverage analysis + +A coverage analysis report is [available +online](https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage). You +can build it yourself: + +``` +# nix build .#hydraJobs.coverage +# xdg-open ./result/coverage/index.html +``` + +Metrics about the change in line/function coverage over time are also +[available](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts). diff -Nru nix-2.8.0/doc/manual/src/expressions/advanced-attributes.md nix-2.16.1+dfsg/doc/manual/src/expressions/advanced-attributes.md --- nix-2.8.0/doc/manual/src/expressions/advanced-attributes.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/advanced-attributes.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,257 +0,0 @@ -# Advanced Attributes - -Derivations can declare some infrequently used optional attributes. - - - `allowedReferences`\ - The optional attribute `allowedReferences` specifies a list of legal - references (dependencies) of the output of the builder. For example, - - ```nix - allowedReferences = []; - ``` - - enforces that the output of a derivation cannot have any runtime - dependencies on its inputs. To allow an output to have a runtime - dependency on itself, use `"out"` as a list item. This is used in - NixOS to check that generated files such as initial ramdisks for - booting Linux don’t have accidental dependencies on other paths in - the Nix store. - - - `allowedRequisites`\ - This attribute is similar to `allowedReferences`, but it specifies - the legal requisites of the whole closure, so all the dependencies - recursively. For example, - - ```nix - allowedRequisites = [ foobar ]; - ``` - - enforces that the output of a derivation cannot have any other - runtime dependency than `foobar`, and in addition it enforces that - `foobar` itself doesn't introduce any other dependency itself. - - - `disallowedReferences`\ - The optional attribute `disallowedReferences` specifies a list of - illegal references (dependencies) of the output of the builder. For - example, - - ```nix - disallowedReferences = [ foo ]; - ``` - - enforces that the output of a derivation cannot have a direct - runtime dependencies on the derivation `foo`. - - - `disallowedRequisites`\ - This attribute is similar to `disallowedReferences`, but it - specifies illegal requisites for the whole closure, so all the - dependencies recursively. For example, - - ```nix - disallowedRequisites = [ foobar ]; - ``` - - enforces that the output of a derivation cannot have any runtime - dependency on `foobar` or any other derivation depending recursively - on `foobar`. - - - `exportReferencesGraph`\ - This attribute allows builders access to the references graph of - their inputs. The attribute is a list of inputs in the Nix store - whose references graph the builder needs to know. The value of - this attribute should be a list of pairs `[ name1 path1 name2 - path2 ... ]`. The references graph of each *pathN* will be stored - in a text file *nameN* in the temporary build directory. The text - files have the format used by `nix-store --register-validity` - (with the deriver fields left empty). For example, when the - following derivation is built: - - ```nix - derivation { - ... - exportReferencesGraph = [ "libfoo-graph" libfoo ]; - }; - ``` - - the references graph of `libfoo` is placed in the file - `libfoo-graph` in the temporary build directory. - - `exportReferencesGraph` is useful for builders that want to do - something with the closure of a store path. Examples include the - builders in NixOS that generate the initial ramdisk for booting - Linux (a `cpio` archive containing the closure of the boot script) - and the ISO-9660 image for the installation CD (which is populated - with a Nix store containing the closure of a bootable NixOS - configuration). - - - `impureEnvVars`\ - This attribute allows you to specify a list of environment variables - that should be passed from the environment of the calling user to - the builder. Usually, the environment is cleared completely when the - builder is executed, but with this attribute you can allow specific - environment variables to be passed unmodified. For example, - `fetchurl` in Nixpkgs has the line - - ```nix - impureEnvVars = [ "http_proxy" "https_proxy" ... ]; - ``` - - to make it use the proxy server configuration specified by the user - in the environment variables `http_proxy` and friends. - - This attribute is only allowed in *fixed-output derivations* (see - below), where impurities such as these are okay since (the hash - of) the output is known in advance. It is ignored for all other - derivations. - - > **Warning** - > - > `impureEnvVars` implementation takes environment variables from - > the current builder process. When a daemon is building its - > environmental variables are used. Without the daemon, the - > environmental variables come from the environment of the - > `nix-build`. - - - `outputHash`; `outputHashAlgo`; `outputHashMode`\ - These attributes declare that the derivation is a so-called - *fixed-output derivation*, which means that a cryptographic hash of - the output is already known in advance. When the build of a - fixed-output derivation finishes, Nix computes the cryptographic - hash of the output and compares it to the hash declared with these - attributes. If there is a mismatch, the build fails. - - The rationale for fixed-output derivations is derivations such as - those produced by the `fetchurl` function. This function downloads a - file from a given URL. To ensure that the downloaded file has not - been modified, the caller must also specify a cryptographic hash of - the file. For example, - - ```nix - fetchurl { - url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz"; - sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; - } - ``` - - It sometimes happens that the URL of the file changes, e.g., because - servers are reorganised or no longer available. We then must update - the call to `fetchurl`, e.g., - - ```nix - fetchurl { - url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; - sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; - } - ``` - - If a `fetchurl` derivation was treated like a normal derivation, the - output paths of the derivation and *all derivations depending on it* - would change. For instance, if we were to change the URL of the - Glibc source distribution in Nixpkgs (a package on which almost all - other packages depend) massive rebuilds would be needed. This is - unfortunate for a change which we know cannot have a real effect as - it propagates upwards through the dependency graph. - - For fixed-output derivations, on the other hand, the name of the - output path only depends on the `outputHash*` and `name` attributes, - while all other attributes are ignored for the purpose of computing - the output path. (The `name` attribute is included because it is - part of the path.) - - As an example, here is the (simplified) Nix expression for - `fetchurl`: - - ```nix - { stdenv, curl }: # The curl program is used for downloading. - - { url, sha256 }: - - stdenv.mkDerivation { - name = baseNameOf (toString url); - builder = ./builder.sh; - buildInputs = [ curl ]; - - # This is a fixed-output derivation; the output must be a regular - # file with SHA256 hash sha256. - outputHashMode = "flat"; - outputHashAlgo = "sha256"; - outputHash = sha256; - - inherit url; - } - ``` - - The `outputHashAlgo` attribute specifies the hash algorithm used to - compute the hash. It can currently be `"sha1"`, `"sha256"` or - `"sha512"`. - - The `outputHashMode` attribute determines how the hash is computed. - It must be one of the following two values: - - - `"flat"`\ - The output must be a non-executable regular file. If it isn’t, - the build fails. The hash is simply computed over the contents - of that file (so it’s equal to what Unix commands like - `sha256sum` or `sha1sum` produce). - - This is the default. - - - `"recursive"`\ - The hash is computed over the NAR archive dump of the output - (i.e., the result of [`nix-store - --dump`](../command-ref/nix-store.md#operation---dump)). In - this case, the output can be anything, including a directory - tree. - - The `outputHash` attribute, finally, must be a string containing - the hash in either hexadecimal or base-32 notation. (See the - [`nix-hash` command](../command-ref/nix-hash.md) for information - about converting to and from base-32 notation.) - - - `__contentAddressed` - If this **experimental** attribute is set to true, then the derivation - outputs will be stored in a content-addressed location rather than the - traditional input-addressed one. - This only has an effect if the `ca-derivation` experimental feature is enabled. - - Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above). - - - `passAsFile`\ - A list of names of attributes that should be passed via files rather - than environment variables. For example, if you have - - ```nix - passAsFile = ["big"]; - big = "a very long string"; - ``` - - then when the builder runs, the environment variable `bigPath` - will contain the absolute path to a temporary file containing `a - very long string`. That is, for any attribute *x* listed in - `passAsFile`, Nix will pass an environment variable `xPath` - holding the path of the file containing the value of attribute - *x*. This is useful when you need to pass large strings to a - builder, since most operating systems impose a limit on the size - of the environment (typically, a few hundred kilobyte). - - - `preferLocalBuild`\ - If this attribute is set to `true` and [distributed building is - enabled](../advanced-topics/distributed-builds.md), then, if - possible, the derivation will be built locally instead of forwarded - to a remote machine. This is appropriate for trivial builders - where the cost of doing a download or remote build would exceed - the cost of building locally. - - - `allowSubstitutes`\ - If this attribute is set to `false`, then Nix will always build this - derivation; it will not try to substitute its outputs. This is - useful for very trivial derivations (such as `writeText` in Nixpkgs) - that are cheaper to build than to substitute from a binary cache. - - > **Note** - > - > You need to have a builder configured which satisfies the - > derivation’s `system` attribute, since the derivation cannot be - > substituted. Thus it is usually a good idea to align `system` with - > `builtins.currentSystem` when setting `allowSubstitutes` to - > `false`. For most trivial derivations this should be the case. diff -Nru nix-2.8.0/doc/manual/src/expressions/arguments-variables.md nix-2.16.1+dfsg/doc/manual/src/expressions/arguments-variables.md --- nix-2.8.0/doc/manual/src/expressions/arguments-variables.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/arguments-variables.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,80 +0,0 @@ -# Arguments and Variables - -The [Nix expression for GNU Hello](expression-syntax.md) is a -function; it is missing some arguments that have to be filled in -somewhere. In the Nix Packages collection this is done in the file -`pkgs/top-level/all-packages.nix`, where all Nix expressions for -packages are imported and called with the appropriate arguments. Here -are some fragments of `all-packages.nix`, with annotations of what -they mean: - -```nix -... - -rec { ① - - hello = import ../applications/misc/hello/ex-1 ② { ③ - inherit fetchurl stdenv perl; - }; - - perl = import ../development/interpreters/perl { ④ - inherit fetchurl stdenv; - }; - - fetchurl = import ../build-support/fetchurl { - inherit stdenv; ... - }; - - stdenv = ...; - -} -``` - -1. This file defines a set of attributes, all of which are concrete - derivations (i.e., not functions). In fact, we define a *mutually - recursive* set of attributes. That is, the attributes can refer to - each other. This is precisely what we want since we want to “plug” - the various packages into each other. - -2. Here we *import* the Nix expression for GNU Hello. The import - operation just loads and returns the specified Nix expression. In - fact, we could just have put the contents of the Nix expression - for GNU Hello in `all-packages.nix` at this point. That would be - completely equivalent, but it would make `all-packages.nix` rather - bulky. - - Note that we refer to `../applications/misc/hello/ex-1`, not - `../applications/misc/hello/ex-1/default.nix`. When you try to - import a directory, Nix automatically appends `/default.nix` to the - file name. - -3. This is where the actual composition takes place. Here we *call* the - function imported from `../applications/misc/hello/ex-1` with a set - containing the things that the function expects, namely `fetchurl`, - `stdenv`, and `perl`. We use inherit again to use the attributes - defined in the surrounding scope (we could also have written - `fetchurl = fetchurl;`, etc.). - - The result of this function call is an actual derivation that can be - built by Nix (since when we fill in the arguments of the function, - what we get is its body, which is the call to `stdenv.mkDerivation` - in the [Nix expression for GNU Hello](expression-syntax.md)). - - > **Note** - > - > Nixpkgs has a convenience function `callPackage` that imports and - > calls a function, filling in any missing arguments by passing the - > corresponding attribute from the Nixpkgs set, like this: - > - > ```nix - > hello = callPackage ../applications/misc/hello/ex-1 { }; - > ``` - > - > If necessary, you can set or override arguments: - > - > ```nix - > hello = callPackage ../applications/misc/hello/ex-1 { stdenv = myStdenv; }; - > ``` - -4. Likewise, we have to instantiate Perl, `fetchurl`, and the standard - environment. diff -Nru nix-2.8.0/doc/manual/src/expressions/build-script.md nix-2.16.1+dfsg/doc/manual/src/expressions/build-script.md --- nix-2.8.0/doc/manual/src/expressions/build-script.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/build-script.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,70 +0,0 @@ -# Build Script - -Here is the builder referenced from Hello's Nix expression (stored in -`pkgs/applications/misc/hello/ex-1/builder.sh`): - -```bash -source $stdenv/setup ① - -PATH=$perl/bin:$PATH ② - -tar xvfz $src ③ -cd hello-* -./configure --prefix=$out ④ -make ⑤ -make install -``` - -The builder can actually be made a lot shorter by using the *generic -builder* functions provided by `stdenv`, but here we write out the build -steps to elucidate what a builder does. It performs the following steps: - -1. When Nix runs a builder, it initially completely clears the - environment (except for the attributes declared in the derivation). - This is done to prevent undeclared inputs from being used in the - build process. If for example the `PATH` contained `/usr/bin`, then - you might accidentally use `/usr/bin/gcc`. - - So the first step is to set up the environment. This is done by - calling the `setup` script of the standard environment. The - environment variable `stdenv` points to the location of the - standard environment being used. (It wasn't specified explicitly - as an attribute in Hello's Nix expression, but `mkDerivation` adds - it automatically.) - -2. Since Hello needs Perl, we have to make sure that Perl is in the - `PATH`. The `perl` environment variable points to the location of - the Perl package (since it was passed in as an attribute to the - derivation), so `$perl/bin` is the directory containing the Perl - interpreter. - -3. Now we have to unpack the sources. The `src` attribute was bound to - the result of fetching the Hello source tarball from the network, so - the `src` environment variable points to the location in the Nix - store to which the tarball was downloaded. After unpacking, we `cd` - to the resulting source directory. - - The whole build is performed in a temporary directory created in - `/tmp`, by the way. This directory is removed after the builder - finishes, so there is no need to clean up the sources afterwards. - Also, the temporary directory is always newly created, so you don't - have to worry about files from previous builds interfering with the - current build. - -4. GNU Hello is a typical Autoconf-based package, so we first have to - run its `configure` script. In Nix every package is stored in a - separate location in the Nix store, for instance - `/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1`. Nix - computes this path by cryptographically hashing all attributes of - the derivation. The path is passed to the builder through the `out` - environment variable. So here we give `configure` the parameter - `--prefix=$out` to cause Hello to be installed in the expected - location. - -5. Finally we build Hello (`make`) and install it into the location - specified by `out` (`make install`). - -If you are wondering about the absence of error checking on the result -of various commands called in the builder: this is because the shell -script is evaluated with Bash's `-e` option, which causes the script to -be aborted if any command fails without an error check. diff -Nru nix-2.8.0/doc/manual/src/expressions/builtin-constants.md nix-2.16.1+dfsg/doc/manual/src/expressions/builtin-constants.md --- nix-2.8.0/doc/manual/src/expressions/builtin-constants.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/builtin-constants.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,20 +0,0 @@ -# Built-in Constants - -Here are the constants built into the Nix expression evaluator: - - - `builtins`\ - The set `builtins` contains all the built-in functions and values. - You can use `builtins` to test for the availability of features in - the Nix installation, e.g., - - ```nix - if builtins ? getEnv then builtins.getEnv "PATH" else "" - ``` - - This allows a Nix expression to fall back gracefully on older Nix - installations that don’t have the desired built-in function. - - - `builtins.currentSystem`\ - The built-in value `currentSystem` evaluates to the Nix platform - identifier for the Nix installation on which the expression is being - evaluated, such as `"i686-linux"` or `"x86_64-darwin"`. diff -Nru nix-2.8.0/doc/manual/src/expressions/builtins-prefix.md nix-2.16.1+dfsg/doc/manual/src/expressions/builtins-prefix.md --- nix-2.8.0/doc/manual/src/expressions/builtins-prefix.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/builtins-prefix.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,16 +0,0 @@ -# Built-in Functions - -This section lists the functions built into the Nix expression -evaluator. (The built-in function `derivation` is discussed above.) -Some built-ins, such as `derivation`, are always in scope of every Nix -expression; you can just access them right away. But to prevent -polluting the namespace too much, most built-ins are not in -scope. Instead, you can access them through the `builtins` built-in -value, which is a set that contains all built-in functions and values. -For instance, `derivation` is also available as `builtins.derivation`. - -
-
derivation attrs; - builtins.derivation attrs
-

derivation is described in - its own section.

diff -Nru nix-2.8.0/doc/manual/src/expressions/builtins-suffix.md nix-2.16.1+dfsg/doc/manual/src/expressions/builtins-suffix.md --- nix-2.8.0/doc/manual/src/expressions/builtins-suffix.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/builtins-suffix.md 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -
diff -Nru nix-2.8.0/doc/manual/src/expressions/derivations.md nix-2.16.1+dfsg/doc/manual/src/expressions/derivations.md --- nix-2.8.0/doc/manual/src/expressions/derivations.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/derivations.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,161 +0,0 @@ -# Derivations - -The most important built-in function is `derivation`, which is used to -describe a single derivation (a build action). It takes as input a set, -the attributes of which specify the inputs of the build. - - - There must be an attribute named `system` whose value must be a - string specifying a Nix system type, such as `"i686-linux"` or - `"x86_64-darwin"`. (To figure out your system type, run `nix -vv - --version`.) The build can only be performed on a machine and - operating system matching the system type. (Nix can automatically - [forward builds for other - platforms](../advanced-topics/distributed-builds.md) by forwarding - them to other machines.) - - - There must be an attribute named `name` whose value must be a - string. This is used as a symbolic name for the package by - `nix-env`, and it is appended to the output paths of the derivation. - - - There must be an attribute named `builder` that identifies the - program that is executed to perform the build. It can be either a - derivation or a source (a local file reference, e.g., - `./builder.sh`). - - - Every attribute is passed as an environment variable to the builder. - Attribute values are translated to environment variables as follows: - - - Strings and numbers are just passed verbatim. - - - A *path* (e.g., `../foo/sources.tar`) causes the referenced file - to be copied to the store; its location in the store is put in - the environment variable. The idea is that all sources should - reside in the Nix store, since all inputs to a derivation should - reside in the Nix store. - - - A *derivation* causes that derivation to be built prior to the - present derivation; its default output path is put in the - environment variable. - - - Lists of the previous types are also allowed. They are simply - concatenated, separated by spaces. - - - `true` is passed as the string `1`, `false` and `null` are - passed as an empty string. - - - The optional attribute `args` specifies command-line arguments to be - passed to the builder. It should be a list. - - - The optional attribute `outputs` specifies a list of symbolic - outputs of the derivation. By default, a derivation produces a - single output path, denoted as `out`. However, derivations can - produce multiple output paths. This is useful because it allows - outputs to be downloaded or garbage-collected separately. For - instance, imagine a library package that provides a dynamic library, - header files, and documentation. A program that links against the - library doesn’t need the header files and documentation at runtime, - and it doesn’t need the documentation at build time. Thus, the - library package could specify: - - ```nix - outputs = [ "lib" "headers" "doc" ]; - ``` - - This will cause Nix to pass environment variables `lib`, `headers` - and `doc` to the builder containing the intended store paths of each - output. The builder would typically do something like - - ```bash - ./configure \ - --libdir=$lib/lib \ - --includedir=$headers/include \ - --docdir=$doc/share/doc - ``` - - for an Autoconf-style package. You can refer to each output of a - derivation by selecting it as an attribute, e.g. - - ```nix - buildInputs = [ pkg.lib pkg.headers ]; - ``` - - The first element of `outputs` determines the *default output*. - Thus, you could also write - - ```nix - buildInputs = [ pkg pkg.headers ]; - ``` - - since `pkg` is equivalent to `pkg.lib`. - -The function `mkDerivation` in the Nixpkgs standard environment is a -wrapper around `derivation` that adds a default value for `system` and -always uses Bash as the builder, to which the supplied builder is passed -as a command-line argument. See the Nixpkgs manual for details. - -The builder is executed as follows: - - - A temporary directory is created under the directory specified by - `TMPDIR` (default `/tmp`) where the build will take place. The - current directory is changed to this directory. - - - The environment is cleared and set to the derivation attributes, as - specified above. - - - In addition, the following variables are set: - - - `NIX_BUILD_TOP` contains the path of the temporary directory for - this build. - - - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the - temporary directory. This is to prevent the builder from - accidentally writing temporary files anywhere else. Doing so - might cause interference by other processes. - - - `PATH` is set to `/path-not-set` to prevent shells from - initialising it to their built-in default value. - - - `HOME` is set to `/homeless-shelter` to prevent programs from - using `/etc/passwd` or the like to find the user's home - directory, which could cause impurity. Usually, when `HOME` is - set, it is used as the location of the home directory, even if - it points to a non-existent path. - - - `NIX_STORE` is set to the path of the top-level Nix store - directory (typically, `/nix/store`). - - - For each output declared in `outputs`, the corresponding - environment variable is set to point to the intended path in the - Nix store for that output. Each output path is a concatenation - of the cryptographic hash of all build inputs, the `name` - attribute and the output name. (The output name is omitted if - it’s `out`.) - - - If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple Nix instances from performing the same - build at the same time. - - - A log of the combined standard output and error is written to - `/nix/var/log/nix`. - - - The builder is executed with the arguments specified by the - attribute `args`. If it exits with exit code 0, it is considered to - have succeeded. - - - The temporary directory is removed (unless the `-K` option was - specified). - - - If the build was successful, Nix scans each output path for - references to input paths by looking for the hash parts of the input - paths. Since these are potential runtime dependencies, Nix registers - them as dependencies of the output paths. - - - After the build, Nix sets the last-modified timestamp on all files - in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to - the default group, and sets the mode of the file to 0444 or 0555 - (i.e., read-only, with execute permission enabled if the file was - originally executable). Note that possible `setuid` and `setgid` - bits are cleared. Setuid and setgid programs are not currently - supported by Nix. This is because the Nix archives used in - deployment have no concept of ownership information, and because it - makes the build result dependent on the user performing the build. diff -Nru nix-2.8.0/doc/manual/src/expressions/expression-language.md nix-2.16.1+dfsg/doc/manual/src/expressions/expression-language.md --- nix-2.8.0/doc/manual/src/expressions/expression-language.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/expression-language.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,12 +0,0 @@ -# Nix Expression Language - -The Nix expression language is a pure, lazy, functional language. Purity -means that operations in the language don't have side-effects (for -instance, there is no variable assignment). Laziness means that -arguments to functions are evaluated only when they are needed. -Functional means that functions are “normal” values that can be passed -around and manipulated in interesting ways. The language is not a -full-featured, general purpose language. Its main job is to describe -packages, compositions of packages, and the variability within packages. - -This section presents the various features of the language. diff -Nru nix-2.8.0/doc/manual/src/expressions/expression-syntax.md nix-2.16.1+dfsg/doc/manual/src/expressions/expression-syntax.md --- nix-2.8.0/doc/manual/src/expressions/expression-syntax.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/expression-syntax.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,93 +0,0 @@ -# Expression Syntax - -Here is a Nix expression for GNU Hello: - -```nix -{ stdenv, fetchurl, perl }: ① - -stdenv.mkDerivation { ② - name = "hello-2.1.1"; ③ - builder = ./builder.sh; ④ - src = fetchurl { ⑤ - url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; - sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; - }; - inherit perl; ⑥ -} -``` - -This file is actually already in the Nix Packages collection in -`pkgs/applications/misc/hello/ex-1/default.nix`. It is customary to -place each package in a separate directory and call the single Nix -expression in that directory `default.nix`. The file has the following -elements (referenced from the figure by number): - -1. This states that the expression is a *function* that expects to be - called with three arguments: `stdenv`, `fetchurl`, and `perl`. They - are needed to build Hello, but we don't know how to build them here; - that's why they are function arguments. `stdenv` is a package that - is used by almost all Nix Packages; it provides a - “standard” environment consisting of the things you would expect - in a basic Unix environment: a C/C++ compiler (GCC, to be precise), - the Bash shell, fundamental Unix tools such as `cp`, `grep`, `tar`, - etc. `fetchurl` is a function that downloads files. `perl` is the - Perl interpreter. - - Nix functions generally have the form `{ x, y, ..., z }: e` where - `x`, `y`, etc. are the names of the expected arguments, and where - *e* is the body of the function. So here, the entire remainder of - the file is the body of the function; when given the required - arguments, the body should describe how to build an instance of - the Hello package. - -2. So we have to build a package. Building something from other stuff - is called a *derivation* in Nix (as opposed to sources, which are - built by humans instead of computers). We perform a derivation by - calling `stdenv.mkDerivation`. `mkDerivation` is a function - provided by `stdenv` that builds a package from a set of - *attributes*. A set is just a list of key/value pairs where each - key is a string and each value is an arbitrary Nix - expression. They take the general form `{ name1 = expr1; ... - nameN = exprN; }`. - -3. The attribute `name` specifies the symbolic name and version of - the package. Nix doesn't really care about these things, but they - are used by for instance `nix-env -q` to show a “human-readable” - name for packages. This attribute is required by `mkDerivation`. - -4. The attribute `builder` specifies the builder. This attribute can - sometimes be omitted, in which case `mkDerivation` will fill in a - default builder (which does a `configure; make; make install`, in - essence). Hello is sufficiently simple that the default builder - would suffice, but in this case, we will show an actual builder - for educational purposes. The value `./builder.sh` refers to the - shell script shown in the [next section](build-script.md), - discussed below. - -5. The builder has to know what the sources of the package are. Here, - the attribute `src` is bound to the result of a call to the - `fetchurl` function. Given a URL and a SHA-256 hash of the expected - contents of the file at that URL, this function builds a derivation - that downloads the file and checks its hash. So the sources are a - dependency that like all other dependencies is built before Hello - itself is built. - - Instead of `src` any other name could have been used, and in fact - there can be any number of sources (bound to different attributes). - However, `src` is customary, and it's also expected by the default - builder (which we don't use in this example). - -6. Since the derivation requires Perl, we have to pass the value of the - `perl` function argument to the builder. All attributes in the set - are actually passed as environment variables to the builder, so - declaring an attribute - - ```nix - perl = perl; - ``` - - will do the trick: it binds an attribute `perl` to the function - argument which also happens to be called `perl`. However, it looks a - bit silly, so there is a shorter syntax. The `inherit` keyword - causes the specified attributes to be bound to whatever variables - with the same name happen to be in scope. diff -Nru nix-2.8.0/doc/manual/src/expressions/generic-builder.md nix-2.16.1+dfsg/doc/manual/src/expressions/generic-builder.md --- nix-2.8.0/doc/manual/src/expressions/generic-builder.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/generic-builder.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,66 +0,0 @@ -# Generic Builder Syntax - -Recall that the [build script for GNU Hello](build-script.md) looked -something like this: - -```bash -PATH=$perl/bin:$PATH -tar xvfz $src -cd hello-* -./configure --prefix=$out -make -make install -``` - -The builders for almost all Unix packages look like this — set up some -environment variables, unpack the sources, configure, build, and -install. For this reason the standard environment provides some Bash -functions that automate the build process. Here is what a builder using -the generic build facilities looks like: - -```bash -buildInputs="$perl" ① - -source $stdenv/setup ② - -genericBuild ③ -``` - -Here is what each line means: - -1. The `buildInputs` variable tells `setup` to use the indicated - packages as “inputs”. This means that if a package provides a `bin` - subdirectory, it's added to `PATH`; if it has a `include` - subdirectory, it's added to GCC's header search path; and so on. - (This is implemented in a modular way: `setup` tries to source the - file `pkg/nix-support/setup-hook` of all dependencies. These “setup - hooks” can then set up whatever environment variables they want; for - instance, the setup hook for Perl sets the `PERL5LIB` environment - variable to contain the `lib/site_perl` directories of all inputs.) - -2. The function `genericBuild` is defined in the file `$stdenv/setup`. - -3. The final step calls the shell function `genericBuild`, which - performs the steps that were done explicitly in the previous build - script. The generic builder is smart enough to figure out whether - to unpack the sources using `gzip`, `bzip2`, etc. It can be - customised in many ways; see the Nixpkgs manual for details. - -Discerning readers will note that the `buildInputs` could just as well -have been set in the Nix expression, like this: - -```nix - buildInputs = [ perl ]; -``` - -The `perl` attribute can then be removed, and the builder becomes even -shorter: - -```bash -source $stdenv/setup -genericBuild -``` - -In fact, `mkDerivation` provides a default builder that looks exactly -like that, so it is actually possible to omit the builder for Hello -entirely. diff -Nru nix-2.8.0/doc/manual/src/expressions/language-constructs.md nix-2.16.1+dfsg/doc/manual/src/expressions/language-constructs.md --- nix-2.8.0/doc/manual/src/expressions/language-constructs.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/language-constructs.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,354 +0,0 @@ -# Language Constructs - -## Recursive sets - -Recursive sets are just normal sets, but the attributes can refer to -each other. For example, - -```nix -rec { - x = y; - y = 123; -}.x -``` - -evaluates to `123`. Note that without `rec` the binding `x = y;` would -refer to the variable `y` in the surrounding scope, if one exists, and -would be invalid if no such variable exists. That is, in a normal -(non-recursive) set, attributes are not added to the lexical scope; in a -recursive set, they are. - -Recursive sets of course introduce the danger of infinite recursion. For -example, the expression - -```nix -rec { - x = y; - y = x; -}.x -``` - -will crash with an `infinite recursion encountered` error message. - -## Let-expressions - -A let-expression allows you to define local variables for an expression. -For instance, - -```nix -let - x = "foo"; - y = "bar"; -in x + y -``` - -evaluates to `"foobar"`. - -## Inheriting attributes - -When defining a set or in a let-expression it is often convenient to -copy variables from the surrounding lexical scope (e.g., when you want -to propagate attributes). This can be shortened using the `inherit` -keyword. For instance, - -```nix -let x = 123; in -{ inherit x; - y = 456; -} -``` - -is equivalent to - -```nix -let x = 123; in -{ x = x; - y = 456; -} -``` - -and both evaluate to `{ x = 123; y = 456; }`. (Note that this works -because `x` is added to the lexical scope by the `let` construct.) It is -also possible to inherit attributes from another set. For instance, in -this fragment from `all-packages.nix`, - -```nix -graphviz = (import ../tools/graphics/graphviz) { - inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xlibs) libXaw; -}; - -xlibs = { - libX11 = ...; - libXaw = ...; - ... -} - -libpng = ...; -libjpg = ...; -... -``` - -the set used in the function call to the function defined in -`../tools/graphics/graphviz` inherits a number of variables from the -surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xlibs` (X11 client-side libraries) set. - -Summarizing the fragment - -```nix -... -inherit x y z; -inherit (src-set) a b c; -... -``` - -is equivalent to - -```nix -... -x = x; y = y; z = z; -a = src-set.a; b = src-set.b; c = src-set.c; -... -``` - -when used while defining local variables in a let-expression or while -defining a set. - -## Functions - -Functions have the following form: - -```nix -pattern: body -``` - -The pattern specifies what the argument of the function must look like, -and binds variables in the body to (parts of) the argument. There are -three kinds of patterns: - - - If a pattern is a single identifier, then the function matches any - argument. Example: - - ```nix - let negate = x: !x; - concat = x: y: x + y; - in if negate true then concat "foo" "bar" else "" - ``` - - Note that `concat` is a function that takes one argument and returns - a function that takes another argument. This allows partial - parameterisation (i.e., only filling some of the arguments of a - function); e.g., - - ```nix - map (concat "foo") [ "bar" "bla" "abc" ] - ``` - - evaluates to `[ "foobar" "foobla" "fooabc" ]`. - - - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a - set containing the listed attributes, and binds the values of those - attributes to variables in the function body. For example, the - function - - ```nix - { x, y, z }: z + y + x - ``` - - can only be called with a set containing exactly the attributes `x`, - `y` and `z`. No other attributes are allowed. If you want to allow - additional arguments, you can use an ellipsis (`...`): - - ```nix - { x, y, z, ... }: z + y + x - ``` - - This works on any set that contains at least the three named - attributes. - - It is possible to provide *default values* for attributes, in - which case they are allowed to be missing. A default value is - specified by writing `name ? e`, where *e* is an arbitrary - expression. For example, - - ```nix - { x, y ? "foo", z ? "bar" }: z + y + x - ``` - - specifies a function that only requires an attribute named `x`, but - optionally accepts `y` and `z`. - - - An `@`-pattern provides a means of referring to the whole value - being matched: - - ```nix - args@{ x, y, z, ... }: z + y + x + args.a - ``` - - but can also be written as: - - ```nix - { x, y, z, ... } @ args: z + y + x + args.a - ``` - - Here `args` is bound to the entire argument, which is further - matched against the pattern `{ x, y, z, - ... }`. `@`-pattern makes mainly sense with an ellipsis(`...`) as - you can access attribute names as `a`, using `args.a`, which was - given as an additional attribute to the function. - - > **Warning** - > - > The `args@` expression is bound to the argument passed to the - > function which means that attributes with defaults that aren't - > explicitly specified in the function call won't cause an - > evaluation error, but won't exist in `args`. - > - > For instance - > - > ```nix - > let - > function = args@{ a ? 23, ... }: args; - > in - > function {} - > ```` - > - > will evaluate to an empty attribute set. - -Note that functions do not have names. If you want to give them a name, -you can bind them to an attribute, e.g., - -```nix -let concat = { x, y }: x + y; -in concat { x = "foo"; y = "bar"; } -``` - -## Conditionals - -Conditionals look like this: - -```nix -if e1 then e2 else e3 -``` - -where *e1* is an expression that should evaluate to a Boolean value -(`true` or `false`). - -## Assertions - -Assertions are generally used to check that certain requirements on or -between features and dependencies hold. They look like this: - -```nix -assert e1; e2 -``` - -where *e1* is an expression that should evaluate to a Boolean value. If -it evaluates to `true`, *e2* is returned; otherwise expression -evaluation is aborted and a backtrace is printed. - -Here is a Nix expression for the Subversion package that shows how -assertions can be used:. - -```nix -{ localServer ? false -, httpServer ? false -, sslSupport ? false -, pythonBindings ? false -, javaSwigBindings ? false -, javahlBindings ? false -, stdenv, fetchurl -, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null -}: - -assert localServer -> db4 != null; ① -assert httpServer -> httpd != null && httpd.expat == expat; ② -assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); ③ -assert pythonBindings -> swig != null && swig.pythonSupport; -assert javaSwigBindings -> swig != null && swig.javaSupport; -assert javahlBindings -> j2sdk != null; - -stdenv.mkDerivation { - name = "subversion-1.1.1"; - ... - openssl = if sslSupport then openssl else null; ④ - ... -} -``` - -The points of interest are: - -1. This assertion states that if Subversion is to have support for - local repositories, then Berkeley DB is needed. So if the Subversion - function is called with the `localServer` argument set to `true` but - the `db4` argument set to `null`, then the evaluation fails. - - Note that `->` is the [logical - implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) - Boolean operation. - -2. This is a more subtle condition: if Subversion is built with Apache - (`httpServer`) support, then the Expat library (an XML library) used - by Subversion should be same as the one used by Apache. This is - because in this configuration Subversion code ends up being linked - with Apache code, and if the Expat libraries do not match, a build- - or runtime link error or incompatibility might occur. - -3. This assertion says that in order for Subversion to have SSL support - (so that it can access `https` URLs), an OpenSSL library must be - passed. Additionally, it says that *if* Apache support is enabled, - then Apache's OpenSSL should match Subversion's. (Note that if - Apache support is not enabled, we don't care about Apache's - OpenSSL.) - -4. The conditional here is not really related to assertions, but is - worth pointing out: it ensures that if SSL support is disabled, then - the Subversion derivation is not dependent on OpenSSL, even if a - non-`null` value was passed. This prevents an unnecessary rebuild of - Subversion if OpenSSL changes. - -## With-expressions - -A *with-expression*, - -```nix -with e1; e2 -``` - -introduces the set *e1* into the lexical scope of the expression *e2*. -For instance, - -```nix -let as = { x = "foo"; y = "bar"; }; -in with as; x + y -``` - -evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes -of `as` to the lexical scope in the expression `x + y`. The most common -use of `with` is in conjunction with the `import` function. E.g., - -```nix -with (import ./definitions.nix); ... -``` - -makes all attributes defined in the file `definitions.nix` available as -if they were defined locally in a `let`-expression. - -The bindings introduced by `with` do not shadow bindings introduced by -other means, e.g. - -```nix -let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... -``` - -establishes the same scope as - -```nix -let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... -``` - -## Comments - -Comments can be single-line, started with a `#` character, or -inline/multi-line, enclosed within `/* ... */`. diff -Nru nix-2.8.0/doc/manual/src/expressions/language-operators.md nix-2.16.1+dfsg/doc/manual/src/expressions/language-operators.md --- nix-2.8.0/doc/manual/src/expressions/language-operators.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/language-operators.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,28 +0,0 @@ -# Operators - -The table below lists the operators in the Nix expression language, in -order of precedence (from strongest to weakest binding). - -| Name | Syntax | Associativity | Description | Precedence | -| ------------------------ | ----------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -| Select | *e* `.` *attrpath* \[ `or` *def* \] | none | Select attribute denoted by the attribute path *attrpath* from set *e*. (An attribute path is a dot-separated list of attribute names.) If the attribute doesn’t exist, return *def* if provided, otherwise abort evaluation. | 1 | -| Application | *e1* *e2* | left | Call function *e1* with argument *e2*. | 2 | -| Arithmetic Negation | `-` *e* | none | Arithmetic negation. | 3 | -| Has Attribute | *e* `?` *attrpath* | none | Test whether set *e* contains the attribute denoted by *attrpath*; return `true` or `false`. | 4 | -| List Concatenation | *e1* `++` *e2* | right | List concatenation. | 5 | -| Multiplication | *e1* `*` *e2*, | left | Arithmetic multiplication. | 6 | -| Division | *e1* `/` *e2* | left | Arithmetic division. | 6 | -| Addition | *e1* `+` *e2* | left | Arithmetic addition. | 7 | -| Subtraction | *e1* `-` *e2* | left | Arithmetic subtraction. | 7 | -| String Concatenation | *string1* `+` *string2* | left | String concatenation. | 7 | -| Not | `!` *e* | none | Boolean negation. | 8 | -| Update | *e1* `//` *e2* | right | Return a set consisting of the attributes in *e1* and *e2* (with the latter taking precedence over the former in case of equally named attributes). | 9 | -| Less Than | *e1* `<` *e2*, | none | Arithmetic/lexicographic comparison. | 10 | -| Less Than or Equal To | *e1* `<=` *e2* | none | Arithmetic/lexicographic comparison. | 10 | -| Greater Than | *e1* `>` *e2* | none | Arithmetic/lexicographic comparison. | 10 | -| Greater Than or Equal To | *e1* `>=` *e2* | none | Arithmetic/lexicographic comparison. | 10 | -| Equality | *e1* `==` *e2* | none | Equality. | 11 | -| Inequality | *e1* `!=` *e2* | none | Inequality. | 11 | -| Logical AND | *e1* `&&` *e2* | left | Logical AND. | 12 | -| Logical OR | *e1* || *e2* | left | Logical OR. | 13 | -| Logical Implication | *e1* `->` *e2* | none | Logical implication (equivalent to !e1 || e2). | 14 | diff -Nru nix-2.8.0/doc/manual/src/expressions/language-values.md nix-2.16.1+dfsg/doc/manual/src/expressions/language-values.md --- nix-2.8.0/doc/manual/src/expressions/language-values.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/language-values.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,251 +0,0 @@ -# Values - -## Simple Values - -Nix has the following basic data types: - - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the result of an expression into a string by - enclosing it in `${...}`, a feature known as *antiquotation*. The - enclosed expression must evaluate to something that can be coerced - into a string (meaning that it must be a string, a path, or a - derivation). For instance, rather than writing - - ```nix - "--with-freetype2-library=" + freetype + "/lib" - ``` - - (where `freetype` is a derivation), you can instead write the more - natural - - ```nix - "--with-freetype2-library=${freetype}/lib" - ``` - - The latter is automatically translated to the former. A more - complicated example (from the Nix expression for - [Qt](http://www.trolltech.com/products/qt)): - - ```nix - configureFlags = " - -system-zlib -system-libpng -system-libjpeg - ${if openglSupport then "-dlopen-opengl - -L${mesa}/lib -I${mesa}/include - -L${libXmu}/lib -I${libXmu}/include" else ""} - ${if threadSupport then "-thread" else "-no-thread"} - "; - ``` - - Note that Nix expressions and strings can be arbitrarily nested; in - this case the outer string contains various antiquotations that - themselves contain strings (e.g., `"-thread"`), some of which in - turn contain expressions (e.g., `${mesa}`). - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Antiquotation (`${expr}`) is supported in indented strings. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. - - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - Numbers are type-compatible: pure integer operations will always - return integers, whereas any operation involving at least one - floating point number will have a floating point number as a result. - - - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at - least one slash to be recognised as such. For instance, `builder.sh` - is not a path: it's parsed as an expression that selects the - attribute `sh` from the variable `builder`. If the file name is - relative, i.e., if it does not begin with a slash, it is made - absolute at parse time relative to the directory of the Nix - expression that contained it. For instance, if a Nix expression in - `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path - is `/foo/xyzzy/fnord.nix`. - - If the first component of a path is a `~`, it is interpreted as if - the rest of the path were relative to the user's home directory. - e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user - whose home directory is `/home/edolstra`. - - Paths can also be specified between angle brackets, e.g. - ``. This means that the directories listed in the - environment variable `NIX_PATH` will be searched for the given file - or directory name. - - Antiquotation is supported in any paths except those in angle brackets. - `./${foo}-${bar}.nix` is a more convenient way of writing - `./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At - least one slash must appear *before* any antiquotations for this to be - recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division - operation. `./a.${foo}/b.${bar}` is a path. - - - *Booleans* with values `true` and `false`. - - - The null value, denoted as `null`. - -## Lists - -Lists are formed by enclosing a whitespace-separated list of values -between square brackets. For example, - -```nix -[ 123 ./foo.nix "abc" (f { x = y; }) ] -``` - -defines a list of four elements, the last being the result of a call to -the function `f`. Note that function calls have to be enclosed in -parentheses. If they had been omitted, e.g., - -```nix -[ 123 ./foo.nix "abc" f { x = y; } ] -``` - -the result would be a list of five elements, the fourth one being a -function and the fifth being a set. - -Note that lists are only lazy in values, and they are strict in length. - -## Sets - -Sets are really the core of the language, since ultimately the Nix -language is all about creating derivations, which are really just sets -of attributes to be passed to build scripts. - -Sets are just a list of name/value pairs (called *attributes*) enclosed -in curly brackets, where each value is an arbitrary expression -terminated by a semicolon. For example: - -```nix -{ x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` - -This defines a set with attributes named `x`, `text`, `y`. The order of -the attributes is irrelevant. An attribute name may only occur once. - -Attributes can be selected from a set using the `.` operator. For -instance, - -```nix -{ a = "Foo"; b = "Bar"; }.a -``` - -evaluates to `"Foo"`. It is possible to provide a default value in an -attribute selection using the `or` keyword. For example, - -```nix -{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" -``` - -will evaluate to `"Xyzzy"` because there is no `c` attribute in the set. - -You can use arbitrary double-quoted strings as attribute names: - -```nix -{ "foo ${bar}" = 123; "nix-1.0" = 456; }."foo ${bar}" -``` - -This will evaluate to `123` (Assuming `bar` is antiquotable). In the -case where an attribute name is just a single antiquotation, the quotes -can be dropped: - -```nix -{ foo = 123; }.${bar} or 456 -``` - -This will evaluate to `123` if `bar` evaluates to `"foo"` when coerced -to a string and `456` otherwise (again assuming `bar` is antiquotable). - -In the special case where an attribute name inside of a set declaration -evaluates to `null` (which is normally an error, as `null` is not -antiquotable), that attribute is simply not added to the set: - -```nix -{ ${if foo then "bar" else null} = true; } -``` - -This will evaluate to `{}` if `foo` evaluates to `false`. - -A set that has a `__functor` attribute whose value is callable (i.e. is -itself a function or a set with a `__functor` attribute whose value is -callable) can be applied as if it were a function, with the set itself -passed in first , e.g., - -```nix -let add = { __functor = self: x: x + self.x; }; - inc = add // { x = 1; }; -in inc 1 -``` - -evaluates to `2`. This can be used to attach metadata to a function -without the caller needing to treat it specially, or to implement a form -of object-oriented programming, for example. diff -Nru nix-2.8.0/doc/manual/src/expressions/simple-building-testing.md nix-2.16.1+dfsg/doc/manual/src/expressions/simple-building-testing.md --- nix-2.8.0/doc/manual/src/expressions/simple-building-testing.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/simple-building-testing.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,61 +0,0 @@ -# Building and Testing - -You can now try to build Hello. Of course, you could do `nix-env -f . -iA -hello`, but you may not want to install a possibly broken package just -yet. The best way to test the package is by using the command -`nix-build`, which builds a Nix expression and creates a symlink named -`result` in the current directory: - -```console -$ nix-build -A hello -building path `/nix/store/632d2b22514d...-hello-2.1.1' -hello-2.1.1/ -hello-2.1.1/intl/ -hello-2.1.1/intl/ChangeLog -... - -$ ls -l result -lrwxrwxrwx ... 2006-09-29 10:43 result -> /nix/store/632d2b22514d...-hello-2.1.1 - -$ ./result/bin/hello -Hello, world! -``` - -The `-A` option selects the `hello` attribute. This is faster than -using the symbolic package name specified by the `name` attribute -(which also happens to be `hello`) and is unambiguous (there can be -multiple packages with the symbolic name `hello`, but there can be -only one attribute in a set named `hello`). - -`nix-build` registers the `./result` symlink as a garbage collection -root, so unless and until you delete the `./result` symlink, the output -of the build will be safely kept on your system. You can use -`nix-build`’s `-o` switch to give the symlink another name. - -Nix has transactional semantics. Once a build finishes successfully, Nix -makes a note of this in its database: it registers that the path denoted -by `out` is now “valid”. If you try to build the derivation again, Nix -will see that the path is already valid and finish immediately. If a -build fails, either because it returns a non-zero exit code, because Nix -or the builder are killed, or because the machine crashes, then the -output paths will not be registered as valid. If you try to build the -derivation again, Nix will remove the output paths if they exist (e.g., -because the builder died half-way through `make -install`) and try again. Note that there is no “negative caching”: Nix -doesn't remember that a build failed, and so a failed build can always -be repeated. This is because Nix cannot distinguish between permanent -failures (e.g., a compiler error due to a syntax error in the source) -and transient failures (e.g., a disk full condition). - -Nix also performs locking. If you run multiple Nix builds -simultaneously, and they try to build the same derivation, the first Nix -instance that gets there will perform the build, while the others block -(or perform other derivations if available) until the build finishes: - -```console -$ nix-build -A hello -waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x' -``` - -So it is always safe to run multiple instances of Nix in parallel (which -isn’t the case with, say, `make`). diff -Nru nix-2.8.0/doc/manual/src/expressions/simple-expression.md nix-2.16.1+dfsg/doc/manual/src/expressions/simple-expression.md --- nix-2.8.0/doc/manual/src/expressions/simple-expression.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/simple-expression.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,23 +0,0 @@ -# A Simple Nix Expression - -This section shows how to add and test the [GNU Hello -package](http://www.gnu.org/software/hello/hello.html) to the Nix -Packages collection. Hello is a program that prints out the text “Hello, -world\!”. - -To add a package to the Nix Packages collection, you generally need to -do three things: - -1. Write a Nix expression for the package. This is a file that - describes all the inputs involved in building the package, such as - dependencies, sources, and so on. - -2. Write a *builder*. This is a shell script that builds the package - from the inputs. (In fact, it can be written in any language, but - typically it's a `bash` shell script.) - -3. Add the package to the file `pkgs/top-level/all-packages.nix`. The - Nix expression written in the first step is a *function*; it - requires other packages in order to build it. In this step you put - it all together, i.e., you call the function with the right - arguments to build the actual package. diff -Nru nix-2.8.0/doc/manual/src/expressions/writing-nix-expressions.md nix-2.16.1+dfsg/doc/manual/src/expressions/writing-nix-expressions.md --- nix-2.8.0/doc/manual/src/expressions/writing-nix-expressions.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/expressions/writing-nix-expressions.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,12 +0,0 @@ -This chapter shows you how to write Nix expressions, which instruct Nix -how to build packages. It starts with a simple example (a Nix expression -for GNU Hello), and then moves on to a more in-depth look at the Nix -expression language. - -> **Note** -> -> This chapter is mostly about the Nix expression language. For more -> extensive information on adding packages to the Nix Packages -> collection (such as functions in the standard environment and coding -> conventions), please consult [its -> manual](http://nixos.org/nixpkgs/manual/). diff -Nru nix-2.8.0/doc/manual/src/glossary.md nix-2.16.1+dfsg/doc/manual/src/glossary.md --- nix-2.8.0/doc/manual/src/glossary.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/glossary.md 2023-06-06 10:55:23.000000000 +0000 @@ -1,62 +1,158 @@ # Glossary - - derivation\ - A description of a build action. The result of a derivation is a + - [derivation]{#gloss-derivation}\ + A description of a build task. The result of a derivation is a store object. Derivations are typically specified in Nix expressions - using the [`derivation` primitive](expressions/derivations.md). These are + using the [`derivation` primitive](./language/derivations.md). These are translated into low-level *store derivations* (implicitly by `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). - - store\ + [derivation]: #gloss-derivation + + - [store derivation]{#gloss-store-derivation}\ + A [derivation] represented as a `.drv` file in the [store]. + It has a [store path], like any [store object]. + + Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` + + See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations. + + [store derivation]: #gloss-store-derivation + + - [instantiate]{#gloss-instantiate}, instantiation\ + Translate a [derivation] into a [store derivation]. + + See [`nix-instantiate`](./command-ref/nix-instantiate.md). + + [instantiate]: #gloss-instantiate + + - [realise]{#gloss-realise}, realisation\ + Ensure a [store path] is [valid][validity]. + + This means either running the `builder` executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter]. + + See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). + + See [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). + + [realise]: #gloss-realise + + - [content-addressed derivation]{#gloss-content-addressed-derivation}\ + A derivation which has the + [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) + attribute set to `true`. + + - [fixed-output derivation]{#gloss-fixed-output-derivation}\ + A derivation which includes the + [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + + - [store]{#gloss-store}\ The location in the file system where store objects live. Typically `/nix/store`. - - store path\ - The location in the file system of a store object, i.e., an + From the perspective of the location where Nix is + invoked, the Nix store can be referred to + as a "_local_" or a "_remote_" one: + + + A [local store]{#gloss-local-store} exists on the filesystem of + the machine where Nix is invoked. You can use other + local stores by passing the `--store` flag to the + `nix` command. Local stores can be used for building derivations. + + + A *remote store* exists anywhere other than the + local filesystem. One example is the `/nix/store` + directory on another machine, accessed via `ssh` or + served by the `nix-serve` Perl script. + + [store]: #gloss-store + [local store]: #gloss-local-store + + - [chroot store]{#gloss-chroot-store}\ + A [local store] whose canonical path is anything other than `/nix/store`. + + - [binary cache]{#gloss-binary-cache}\ + A *binary cache* is a Nix store which uses a different format: its + metadata and signatures are kept in `.narinfo` files rather than in a + [Nix database]. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). + + - [store path]{#gloss-store-path}\ + The location of a [store object] in the file system, i.e., an immediate child of the Nix store directory. - - store object\ + Example: `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` + + [store path]: #gloss-store-path + + - [store object]{#gloss-store-object}\ A file that is an immediate child of the Nix store directory. These can be regular files, but also entire directory trees. Store objects can be sources (objects copied from outside of the store), - derivation outputs (objects produced by running a build action), or - derivations (files describing a build action). + derivation outputs (objects produced by running a build task), or + derivations (files describing a build task). + + [store object]: #gloss-store-object - - substitute\ - A substitute is a command invocation stored in the Nix database that + - [input-addressed store object]{#gloss-input-addressed-store-object}\ + A store object produced by building a + non-[content-addressed](#gloss-content-addressed-derivation), + non-[fixed-output](#gloss-fixed-output-derivation) + derivation. + + - [output-addressed store object]{#gloss-output-addressed-store-object}\ + A [store object] whose [store path] is determined by its contents. + This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + + - [substitute]{#gloss-substitute}\ + A substitute is a command invocation stored in the [Nix database] that describes how to build a store object, bypassing the normal build mechanism (i.e., derivations). Typically, the substitute builds the store object by downloading a pre-built version of the store object from some server. - - purity\ + - [substituter]{#gloss-substituter}\ + A *substituter* is an additional store from which Nix will + copy store objects it doesn't have. For details, see the + [`substituters` option](./command-ref/conf-file.md#conf-substituters). + + [substituter]: #gloss-substituter + + - [purity]{#gloss-purity}\ The assumption that equal Nix derivations when run always produce the same output. This cannot be guaranteed in general (e.g., a builder can rely on external inputs such as the network or the system time) but the Nix model assumes it. - - Nix expression\ + - [Nix database]{#gloss-nix-database}\ + An SQlite database to track [reference]s between [store object]s. + This is an implementation detail of the [local store]. + + Default location: `/nix/var/nix/db`. + + [Nix database]: #gloss-nix-database + + - [Nix expression]{#gloss-nix-expression}\ A high-level description of software packages and compositions thereof. Deploying software using Nix entails writing Nix expressions for your packages. Nix expressions are translated to derivations that are stored in the Nix store. These derivations can then be built. - - reference\ - A store path `P` is said to have a reference to a store path `Q` if - the store object at `P` contains the path `Q` somewhere. The - *references* of a store path are the set of store paths to which it - has a reference. - - A derivation can reference other derivations and sources (but not - output paths), whereas an output path only references other output - paths. + - [reference]{#gloss-reference}\ + A [store object] `O` is said to have a *reference* to a store object `P` if a [store path] to `P` appears in the contents of `O`. + + Store objects can refer to both other store objects and themselves. + References from a store object to itself are called *self-references*. + References other than a self-reference must not form a cycle. - - reachable\ + [reference]: #gloss-reference + + - [reachable]{#gloss-reachable}\ A store path `Q` is reachable from another store path `P` if `Q` is in the *closure* of the *references* relation. - - closure\ + - [closure]{#gloss-closure}\ The closure of a store path is the set of store paths that are directly or indirectly “reachable” from that store path; that is, it’s the closure of the path under the *references* relation. For @@ -64,41 +160,71 @@ build-time dependencies, while the closure of its output path is equivalent to its runtime dependencies. For correct deployment it is necessary to deploy whole closures, since otherwise at runtime - files could be missing. The command `nix-store -qR` prints out + files could be missing. The command `nix-store --query --requisites ` prints out closures of store paths. - As an example, if the store object at path `P` contains a reference - to path `Q`, then `Q` is in the closure of `P`. Further, if `Q` + As an example, if the [store object] at path `P` contains a [reference] + to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` references `R` then `R` is also in the closure of `P`. - - output path\ - A store path produced by a derivation. + [closure]: #gloss-closure + + - [output path]{#gloss-output-path}\ + A [store path] produced by a [derivation]. + + [output path]: #gloss-output-path + + - [deriver]{#gloss-deriver}\ + The [store derivation] that produced an [output path]. + + - [validity]{#gloss-validity}\ + A store path is valid if all [store object]s in its [closure] can be read from the [store]. - - deriver\ - The deriver of an *output path* is the store - derivation that built it. - - - validity\ - A store path is considered *valid* if it exists in the file system, - is listed in the Nix database as being valid, and if all paths in - its closure are also valid. + For a [local store], this means: + - The store path leads to an existing [store object] in that [store]. + - The store path is listed in the [Nix database] as being valid. + - All paths in the store path's [closure] are valid. - - user environment\ + [validity]: #gloss-validity + + - [user environment]{#gloss-user-env}\ An automatically generated store object that consists of a set of symlinks to “active” applications, i.e., other store paths. These are generated automatically by - [`nix-env`](command-ref/nix-env.md). See *profiles*. + [`nix-env`](./command-ref/nix-env.md). See *profiles*. - - profile\ + - [profile]{#gloss-profile}\ A symlink to the current *user environment* of a user, e.g., `/nix/var/nix/profiles/default`. - - NAR\ + - [installable]{#gloss-installable}\ + Something that can be realised in the Nix store. + + See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. + + - [NAR]{#gloss-nar}\ A *N*ix *AR*chive. This is a serialisation of a path in the Nix store. It can contain regular files, directories and symbolic links. NARs are generated and unpacked using `nix-store --dump` and `nix-store --restore`. - - `∅` \ + + - [`∅`]{#gloss-emtpy-set}\ The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. - - `ε` \ + + - [`ε`]{#gloss-epsilon}\ The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. + + - [string interpolation]{#gloss-string-interpolation}\ + Expanding expressions enclosed in `${ }` within a [string], [path], or [attribute name]. + + See [String interpolation](./language/string-interpolation.md) for details. + + [string]: ./language/values.md#type-string + [path]: ./language/values.md#type-path + [attribute name]: ./language/values.md#attribute-set + + - [experimental feature]{#gloss-experimental-feature}\ + Not yet stabilized functionality guarded by named experimental feature flags. + These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. + + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). diff -Nru nix-2.8.0/doc/manual/src/installation/env-variables.md nix-2.16.1+dfsg/doc/manual/src/installation/env-variables.md --- nix-2.8.0/doc/manual/src/installation/env-variables.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/installation/env-variables.md 2023-06-06 10:55:23.000000000 +0000 @@ -27,7 +27,7 @@ ```console $ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt -$ sh <(curl -L https://nixos.org/nix/install) +$ curl -L https://nixos.org/nix/install | sh ``` In the shell profile and rc files (for example, `/etc/bashrc`, @@ -38,18 +38,15 @@ ``` > **Note** -> +> > You must not add the export and then do the install, as the Nix > installer will detect the presence of Nix configuration, and abort. -## `NIX_SSL_CERT_FILE` with macOS and the Nix daemon +If you use the Nix daemon, you should also add the following to +`/etc/nix/nix.conf`: -On macOS you must specify the environment variable for the Nix daemon -service, then restart it: - -```console -$ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt -$ sudo launchctl kickstart -k system/org.nixos.nix-daemon +``` +ssl-cert-file = /etc/ssl/my-certificate-bundle.crt ``` ## Proxy Environment Variables diff -Nru nix-2.8.0/doc/manual/src/installation/installation.md nix-2.16.1+dfsg/doc/manual/src/installation/installation.md --- nix-2.8.0/doc/manual/src/installation/installation.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/installation/installation.md 2023-06-06 10:55:23.000000000 +0000 @@ -1,2 +1,38 @@ -This section describes how to install and configure Nix for first-time -use. +# Installation + +This section describes how to install and configure Nix for first-time use. + +The current recommended option on Linux and MacOS is [multi-user](#multi-user). + +## Multi-user + +This installation offers better sharing, improved isolation, and more security +over a single user installation. + +This option requires either: + +* Linux running systemd, with SELinux disabled +* MacOS + +```console +$ bash <(curl -L https://nixos.org/nix/install) --daemon +``` + +## Single-user + +> Single-user is not supported on Mac. + +This installation has less requirements than the multi-user install, however it +cannot offer equivalent sharing, isolation, or security. + +This option is suitable for systems without systemd. + +```console +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon +``` + +## Distributions + +The Nix community maintains installers for several distributions. + +They can be found in the [`nix-community/nix-installers`](https://github.com/nix-community/nix-installers) repository. diff -Nru nix-2.8.0/doc/manual/src/installation/installing-binary.md nix-2.16.1+dfsg/doc/manual/src/installation/installing-binary.md --- nix-2.8.0/doc/manual/src/installation/installing-binary.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/installation/installing-binary.md 2023-06-06 10:55:23.000000000 +0000 @@ -3,7 +3,7 @@ The easiest way to install Nix is to run the following command: ```console -$ sh <(curl -L https://nixos.org/nix/install) +$ curl -L https://nixos.org/nix/install | sh ``` This will run the installer interactively (causing it to explain what @@ -13,7 +13,7 @@ - multi-user on macOS > **Notes on read-only filesystem root in macOS 10.15 Catalina +** - > + > > - It took some time to support this cleanly. You may see posts, > examples, and tutorials using obsolete workarounds. > - Supporting it cleanly made macOS installs too complex to qualify @@ -27,12 +27,12 @@ To explicitly select a single-user installation on your system: ```console -$ sh <(curl -L https://nixos.org/nix/install) --no-daemon +$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon ``` This will perform a single-user installation of Nix, meaning that `/nix` -is owned by the invoking user. You should run this under your usual user -account, *not* as root. The script will invoke `sudo` to create `/nix` +is owned by the invoking user. You can run this under your usual user +account or root. The script will invoke `sudo` to create `/nix` if it doesn’t already exist. If you don’t have `sudo`, you should manually create `/nix` first as root, e.g.: @@ -47,12 +47,6 @@ `NIX_INSTALLER_NO_MODIFY_PROFILE` environment variable before executing the install script to disable this behaviour. -You can uninstall Nix simply by running: - -```console -$ rm -rf /nix -``` - # Multi User Installation The multi-user Nix installation creates system users, and a system @@ -66,16 +60,16 @@ your system: ```console -$ sh <(curl -L https://nixos.org/nix/install) --daemon +$ curl -L https://nixos.org/nix/install | sh -s -- --daemon ``` The multi-user installation of Nix will create build users between the user IDs 30001 and 30032, and a group with the group ID 30000. You -should run this under your usual user account, *not* as root. The script +can run this under your usual user account or root. The script will invoke `sudo` as needed. > **Note** -> +> > If you need Nix to use a different group ID or user ID set, you will > have to download the tarball manually and [edit the install > script](#installing-from-a-binary-tarball). @@ -84,114 +78,13 @@ The installer will first back up these files with a `.backup-before-nix` extension. The installer will also create `/etc/profile.d/nix.sh`. -## Uninstalling - -### Linux - -```console -sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels - -# If you are on Linux with systemd, you will need to run: -sudo systemctl stop nix-daemon.socket -sudo systemctl stop nix-daemon.service -sudo systemctl disable nix-daemon.socket -sudo systemctl disable nix-daemon.service -sudo systemctl daemon-reload -``` - -There may also be references to Nix in `/etc/profile`, `/etc/bashrc`, -and `/etc/zshrc` which you may remove. - -### macOS - -1. Edit `/etc/zshrc` and `/etc/bashrc` to remove the lines sourcing - `nix-daemon.sh`, which should look like this: - - ```bash - # Nix - if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then - . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' - fi - # End Nix - ``` - - If these files haven't been altered since installing Nix you can simply put - the backups back in place: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - ``` - - This will stop shells from sourcing the file and bringing everything you - installed using Nix in scope. +# macOS Installation -2. Stop and remove the Nix daemon services: - - ```console - sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist - sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist - sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist - sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist - ``` - - This stops the Nix daemon and prevents it from being started next time you - boot the system. - -3. Remove the `nixbld` group and the `_nixbuildN` users: - - ```console - sudo dscl . -delete /Groups/nixbld - for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done - ``` - - This will remove all the build users that no longer serve a purpose. - -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like this, - `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic - mounting of the Nix Store volume. - -5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only - line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. - This will prevent the creation of the empty `/nix` directory to provide a - mountpoint for the Nix Store volume. - -6. Remove the files Nix added to your system: - - ```console - sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels - ``` - - This gets rid of any data Nix may have created except for the store which is - removed next. - -7. Remove the Nix Store volume: - - ```console - sudo diskutil apfs deleteVolume /nix - ``` - - This will remove the Nix Store volume and everything that was added to the - store. - -> **Note** -> -> After you complete the steps here, you will still have an empty `/nix` -> directory. This is an expected sign of a successful uninstall. The empty -> `/nix` directory will disappear the next time you reboot. -> -> You do not have to reboot to finish uninstalling Nix. The uninstall is -> complete. macOS (Catalina+) directly controls root directories and its -> read-only root will prevent you from manually deleting the empty `/nix` -> mountpoint. - -# macOS Installation +[]{#sect-macos-installation-change-store-prefix}[]{#sect-macos-installation-encrypted-volume}[]{#sect-macos-installation-symlink}[]{#sect-macos-installation-recommended-notes} We believe we have ironed out how to cleanly support the read-only root -on modern macOS. New installs will do this automatically, and you can -also re-run a new installer to convert your existing setup. +on modern macOS. New installs will do this automatically. This section previously detailed the situation, options, and trade-offs, but it now only outlines what the installer does. You don't need to know @@ -235,19 +128,16 @@ # Installing a pinned Nix version from a URL -NixOS.org hosts version-specific installation URLs for all Nix versions -since 1.11.16, at `https://releases.nixos.org/nix/nix-version/install`. +Version-specific installation URLs for all Nix versions +since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/). +The corresponding SHA-256 hash can be found in the directory for the given version. -These install scripts can be used the same as the main NixOS.org -installation script: +These install scripts can be used the same as usual: ```console -$ sh <(curl -L https://nixos.org/nix/install) +$ curl -L https://releases.nixos.org/nix/nix-/install | sh ``` -In the same directory of the install script are sha256 sums, and gpg -signature files. - # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its diff -Nru nix-2.8.0/doc/manual/src/installation/prerequisites-source.md nix-2.16.1+dfsg/doc/manual/src/installation/prerequisites-source.md --- nix-2.8.0/doc/manual/src/installation/prerequisites-source.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/installation/prerequisites-source.md 2023-06-06 10:55:23.000000000 +0000 @@ -71,3 +71,8 @@ . This is an optional dependency and can be disabled by providing a `--disable-cpuid` to the `configure` script. + + - Unless `./configure --disable-tests` is specified, GoogleTest (GTest) and + RapidCheck are required, which are available at + and + respectively. diff -Nru nix-2.8.0/doc/manual/src/installation/uninstall.md nix-2.16.1+dfsg/doc/manual/src/installation/uninstall.md --- nix-2.8.0/doc/manual/src/installation/uninstall.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/installation/uninstall.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,148 @@ +# Uninstalling Nix + +## Single User + +If you have a [single-user installation](./installing-binary.md#single-user-installation) of Nix, uninstall it by running: + +```console +$ rm -rf /nix +``` + +## Multi User + +Removing a [multi-user installation](./installing-binary.md#multi-user-installation) of Nix is more involved, and depends on the operating system. + +### Linux + +If you are on Linux with systemd: + +1. Remove the Nix daemon service: + + ```console + sudo systemctl stop nix-daemon.service + sudo systemctl disable nix-daemon.socket nix-daemon.service + sudo systemctl daemon-reload + ``` + +Remove files created by Nix: + +```console +sudo rm -rf /etc/nix /etc/profile.d/nix.sh /etc/tmpfiles.d/nix-daemon.conf /nix ~root/.nix-channels ~root/.nix-defexpr ~root/.nix-profile +``` + +Remove build users and their group: + +```console +for i in $(seq 1 32); do + sudo userdel nixbld$i +done +sudo groupdel nixbld +``` + +There may also be references to Nix in + +- `/etc/bash.bashrc` +- `/etc/bashrc` +- `/etc/profile` +- `/etc/zsh/zshrc` +- `/etc/zshrc` + +which you may remove. + +### macOS + +1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: + + ```bash + # Nix + if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then + . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' + fi + # End Nix + ``` + + If these files haven't been altered since installing Nix you can simply put + the backups back in place: + + ```console + sudo mv /etc/zshrc.backup-before-nix /etc/zshrc + sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc + ``` + + This will stop shells from sourcing the file and bringing everything you + installed using Nix in scope. + +2. Stop and remove the Nix daemon services: + + ```console + sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist + sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist + sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist + sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist + ``` + + This stops the Nix daemon and prevents it from being started next time you + boot the system. + +3. Remove the `nixbld` group and the `_nixbuildN` users: + + ```console + sudo dscl . -delete /Groups/nixbld + for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done + ``` + + This will remove all the build users that no longer serve a purpose. + +4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store + volume on `/nix`, which looks like + `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or + `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic + mounting of the Nix Store volume. + +5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only + line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. + This will prevent the creation of the empty `/nix` directory to provide a + mountpoint for the Nix Store volume. + +6. Remove the files Nix added to your system: + + ```console + sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels + ``` + + This gets rid of any data Nix may have created except for the store which is + removed next. + +7. Remove the Nix Store volume: + + ```console + sudo diskutil apfs deleteVolume /nix + ``` + + This will remove the Nix Store volume and everything that was added to the + store. + + If the output indicates that the command couldn't remove the volume, you should + make sure you don't have an _unmounted_ Nix Store volume. Look for a + "Nix Store" volume in the output of the following command: + + ```console + diskutil list + ``` + + If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil + deleteVolume command, but replace `/nix` with the store volume's `diskXsY` + identifier. + +> **Note** +> +> After you complete the steps here, you will still have an empty `/nix` +> directory. This is an expected sign of a successful uninstall. The empty +> `/nix` directory will disappear the next time you reboot. +> +> You do not have to reboot to finish uninstalling Nix. The uninstall is +> complete. macOS (Catalina+) directly controls root directories and its +> read-only root will prevent you from manually deleting the empty `/nix` +> mountpoint. + diff -Nru nix-2.8.0/doc/manual/src/installation/upgrading.md nix-2.16.1+dfsg/doc/manual/src/installation/upgrading.md --- nix-2.8.0/doc/manual/src/installation/upgrading.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/installation/upgrading.md 2023-06-06 10:55:23.000000000 +0000 @@ -2,13 +2,13 @@ Multi-user Nix users on macOS can upgrade Nix by running: `sudo -i sh -c 'nix-channel --update && -nix-env -iA nixpkgs.nix && +nix-env --install --attr nixpkgs.nix && launchctl remove org.nixos.nix-daemon && launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'` Single-user installations of Nix should run this: `nix-channel --update; -nix-env -iA nixpkgs.nix nixpkgs.cacert` +nix-env --install --attr nixpkgs.nix nixpkgs.cacert` Multi-user Nix users on Linux should run this with sudo: `nix-channel ---update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl +--update; nix-env --install --attr nixpkgs.nix nixpkgs.cacert; systemctl daemon-reload; systemctl restart nix-daemon` diff -Nru nix-2.8.0/doc/manual/src/introduction.md nix-2.16.1+dfsg/doc/manual/src/introduction.md --- nix-2.8.0/doc/manual/src/introduction.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/introduction.md 2023-06-06 10:55:23.000000000 +0000 @@ -76,7 +76,7 @@ old version: ```console -$ nix-env --upgrade -A nixpkgs.some-package +$ nix-env --upgrade --attr nixpkgs.some-package $ nix-env --rollback ``` @@ -104,7 +104,7 @@ Packages are built from _Nix expressions_, which is a simple functional language. A Nix expression describes everything that goes -into a package build action (a “derivation”): other packages, sources, +into a package build task (a “derivation”): other packages, sources, the build script, environment variables for the build script, etc. Nix tries very hard to ensure that Nix expressions are _deterministic_: building a Nix expression twice should yield the same @@ -122,7 +122,7 @@ source, so an installation action like ```console -$ nix-env --install -A nixpkgs.firefox +$ nix-env --install --attr nixpkgs.firefox ``` _could_ cause quite a bit of build activity, as not only Firefox but @@ -158,7 +158,7 @@ Nix expression](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/newsreaders/pan/default.nix): ```console -$ nix-shell '' -A pan +$ nix-shell '' --attr pan ``` You’re then dropped into a shell where you can edit, build and test diff -Nru nix-2.8.0/doc/manual/src/language/advanced-attributes.md nix-2.16.1+dfsg/doc/manual/src/language/advanced-attributes.md --- nix-2.8.0/doc/manual/src/language/advanced-attributes.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/language/advanced-attributes.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,347 @@ +# Advanced Attributes + +Derivations can declare some infrequently used optional attributes. + + - [`allowedReferences`]{#adv-attr-allowedReferences}\ + The optional attribute `allowedReferences` specifies a list of legal + references (dependencies) of the output of the builder. For example, + + ```nix + allowedReferences = []; + ``` + + enforces that the output of a derivation cannot have any runtime + dependencies on its inputs. To allow an output to have a runtime + dependency on itself, use `"out"` as a list item. This is used in + NixOS to check that generated files such as initial ramdisks for + booting Linux don’t have accidental dependencies on other paths in + the Nix store. + + - [`allowedRequisites`]{#adv-attr-allowedRequisites}\ + This attribute is similar to `allowedReferences`, but it specifies + the legal requisites of the whole closure, so all the dependencies + recursively. For example, + + ```nix + allowedRequisites = [ foobar ]; + ``` + + enforces that the output of a derivation cannot have any other + runtime dependency than `foobar`, and in addition it enforces that + `foobar` itself doesn't introduce any other dependency itself. + + - [`disallowedReferences`]{#adv-attr-disallowedReferences}\ + The optional attribute `disallowedReferences` specifies a list of + illegal references (dependencies) of the output of the builder. For + example, + + ```nix + disallowedReferences = [ foo ]; + ``` + + enforces that the output of a derivation cannot have a direct + runtime dependencies on the derivation `foo`. + + - [`disallowedRequisites`]{#adv-attr-disallowedRequisites}\ + This attribute is similar to `disallowedReferences`, but it + specifies illegal requisites for the whole closure, so all the + dependencies recursively. For example, + + ```nix + disallowedRequisites = [ foobar ]; + ``` + + enforces that the output of a derivation cannot have any runtime + dependency on `foobar` or any other derivation depending recursively + on `foobar`. + + - [`exportReferencesGraph`]{#adv-attr-exportReferencesGraph}\ + This attribute allows builders access to the references graph of + their inputs. The attribute is a list of inputs in the Nix store + whose references graph the builder needs to know. The value of + this attribute should be a list of pairs `[ name1 path1 name2 + path2 ... ]`. The references graph of each *pathN* will be stored + in a text file *nameN* in the temporary build directory. The text + files have the format used by `nix-store --register-validity` + (with the deriver fields left empty). For example, when the + following derivation is built: + + ```nix + derivation { + ... + exportReferencesGraph = [ "libfoo-graph" libfoo ]; + }; + ``` + + the references graph of `libfoo` is placed in the file + `libfoo-graph` in the temporary build directory. + + `exportReferencesGraph` is useful for builders that want to do + something with the closure of a store path. Examples include the + builders in NixOS that generate the initial ramdisk for booting + Linux (a `cpio` archive containing the closure of the boot script) + and the ISO-9660 image for the installation CD (which is populated + with a Nix store containing the closure of a bootable NixOS + configuration). + + - [`impureEnvVars`]{#adv-attr-impureEnvVars}\ + This attribute allows you to specify a list of environment variables + that should be passed from the environment of the calling user to + the builder. Usually, the environment is cleared completely when the + builder is executed, but with this attribute you can allow specific + environment variables to be passed unmodified. For example, + `fetchurl` in Nixpkgs has the line + + ```nix + impureEnvVars = [ "http_proxy" "https_proxy" ... ]; + ``` + + to make it use the proxy server configuration specified by the user + in the environment variables `http_proxy` and friends. + + This attribute is only allowed in *fixed-output derivations* (see + below), where impurities such as these are okay since (the hash + of) the output is known in advance. It is ignored for all other + derivations. + + > **Warning** + > + > `impureEnvVars` implementation takes environment variables from + > the current builder process. When a daemon is building its + > environmental variables are used. Without the daemon, the + > environmental variables come from the environment of the + > `nix-build`. + + - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ + These attributes declare that the derivation is a so-called + *fixed-output derivation*, which means that a cryptographic hash of + the output is already known in advance. When the build of a + fixed-output derivation finishes, Nix computes the cryptographic + hash of the output and compares it to the hash declared with these + attributes. If there is a mismatch, the build fails. + + The rationale for fixed-output derivations is derivations such as + those produced by the `fetchurl` function. This function downloads a + file from a given URL. To ensure that the downloaded file has not + been modified, the caller must also specify a cryptographic hash of + the file. For example, + + ```nix + fetchurl { + url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz"; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; + } + ``` + + It sometimes happens that the URL of the file changes, e.g., because + servers are reorganised or no longer available. We then must update + the call to `fetchurl`, e.g., + + ```nix + fetchurl { + url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; + } + ``` + + If a `fetchurl` derivation was treated like a normal derivation, the + output paths of the derivation and *all derivations depending on it* + would change. For instance, if we were to change the URL of the + Glibc source distribution in Nixpkgs (a package on which almost all + other packages depend) massive rebuilds would be needed. This is + unfortunate for a change which we know cannot have a real effect as + it propagates upwards through the dependency graph. + + For fixed-output derivations, on the other hand, the name of the + output path only depends on the `outputHash*` and `name` attributes, + while all other attributes are ignored for the purpose of computing + the output path. (The `name` attribute is included because it is + part of the path.) + + As an example, here is the (simplified) Nix expression for + `fetchurl`: + + ```nix + { stdenv, curl }: # The curl program is used for downloading. + + { url, sha256 }: + + stdenv.mkDerivation { + name = baseNameOf (toString url); + builder = ./builder.sh; + buildInputs = [ curl ]; + + # This is a fixed-output derivation; the output must be a regular + # file with SHA256 hash sha256. + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = sha256; + + inherit url; + } + ``` + + The `outputHashAlgo` attribute specifies the hash algorithm used to + compute the hash. It can currently be `"sha1"`, `"sha256"` or + `"sha512"`. + + The `outputHashMode` attribute determines how the hash is computed. + It must be one of the following two values: + + - `"flat"`\ + The output must be a non-executable regular file. If it isn’t, + the build fails. The hash is simply computed over the contents + of that file (so it’s equal to what Unix commands like + `sha256sum` or `sha1sum` produce). + + This is the default. + + - `"recursive"`\ + The hash is computed over the NAR archive dump of the output + (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In + this case, the output can be anything, including a directory + tree. + + The `outputHash` attribute, finally, must be a string containing + the hash in either hexadecimal or base-32 notation. (See the + [`nix-hash` command](../command-ref/nix-hash.md) for information + about converting to and from base-32 notation.) + + - [`__contentAddressed`]{#adv-attr-__contentAddressed} + > **Warning** + > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > + > To use this attribute, you must enable the + > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > For example, in [nix.conf](../command-ref/conf-file.md) you could add: + > + > ``` + > extra-experimental-features = ca-derivations + > ``` + + If this attribute is set to `true`, then the derivation + outputs will be stored in a content-addressed location rather than the + traditional input-addressed one. + + Setting this attribute also requires setting + [`outputHashMode`](#adv-attr-outputHashMode) + and + [`outputHashAlgo`](#adv-attr-outputHashAlgo) + like for *fixed-output derivations* (see above). + + - [`passAsFile`]{#adv-attr-passAsFile}\ + A list of names of attributes that should be passed via files rather + than environment variables. For example, if you have + + ```nix + passAsFile = ["big"]; + big = "a very long string"; + ``` + + then when the builder runs, the environment variable `bigPath` + will contain the absolute path to a temporary file containing `a + very long string`. That is, for any attribute *x* listed in + `passAsFile`, Nix will pass an environment variable `xPath` + holding the path of the file containing the value of attribute + *x*. This is useful when you need to pass large strings to a + builder, since most operating systems impose a limit on the size + of the environment (typically, a few hundred kilobyte). + + - [`preferLocalBuild`]{#adv-attr-preferLocalBuild}\ + If this attribute is set to `true` and [distributed building is + enabled](../advanced-topics/distributed-builds.md), then, if + possible, the derivation will be built locally instead of forwarded + to a remote machine. This is appropriate for trivial builders + where the cost of doing a download or remote build would exceed + the cost of building locally. + + - [`allowSubstitutes`]{#adv-attr-allowSubstitutes}\ + If this attribute is set to `false`, then Nix will always build this + derivation; it will not try to substitute its outputs. This is + useful for very trivial derivations (such as `writeText` in Nixpkgs) + that are cheaper to build than to substitute from a binary cache. + + > **Note** + > + > You need to have a builder configured which satisfies the + > derivation’s `system` attribute, since the derivation cannot be + > substituted. Thus it is usually a good idea to align `system` with + > `builtins.currentSystem` when setting `allowSubstitutes` to + > `false`. For most trivial derivations this should be the case. + + - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ + If the special attribute `__structuredAttrs` is set to `true`, the other derivation + attributes are serialised in JSON format and made available to the + builder via the file `.attrs.json` in the builder’s temporary + directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files + have no size restrictions, unlike process environments. + + It also makes it possible to tweak derivation settings in a structured way; see + [`outputChecks`](#adv-attr-outputChecks) for example. + + As a convenience to Bash builders, + Nix writes a script named `.attrs.sh` to the builder’s directory + that initialises shell variables corresponding to all attributes + that are representable in Bash. This includes non-nested + (associative) arrays. For example, the attribute `hardening.format = true` + ends up as the Bash associative array element `${hardening[format]}`. + + - [`outputChecks`]{#adv-attr-outputChecks}\ + When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` + attribute allows defining checks per-output. + + In addition to + [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), + the following attributes are available: + + - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). + - `maxClosureSize` defines the maximum size of the output's closure. + - `ignoreSelfRefs` controls whether self-references should be considered when + checking for allowed references/requisites. + + Example: + + ```nix + __structuredAttrs = true; + + outputChecks.out = { + # The closure of 'out' must not be larger than 256 MiB. + maxClosureSize = 256 * 1024 * 1024; + + # It must not refer to the C compiler or to the 'dev' output. + disallowedRequisites = [ stdenv.cc "dev" ]; + }; + + outputChecks.dev = { + # The 'dev' output must not be larger than 128 KiB. + maxSize = 128 * 1024; + }; + ``` + + - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ + > **Warning** + > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > + > To use this attribute, you must enable the + > [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature. + > For example, in [nix.conf](../command-ref/conf-file.md) you could add: + > + > ``` + > extra-experimental-features = discard-references + > ``` + + When using [structured attributes](#adv-attr-structuredAttrs), the + attribute `unsafeDiscardReferences` is an attribute set with a boolean value for each output name. + If set to `true`, it disables scanning the output for runtime dependencies. + + Example: + + ```nix + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + ``` + + This is useful, for example, when generating self-contained filesystem images with + their own embedded Nix store: hashes found inside such an image refer + to the embedded store and not to the host's Nix store. diff -Nru nix-2.8.0/doc/manual/src/language/builtin-constants.md nix-2.16.1+dfsg/doc/manual/src/language/builtin-constants.md --- nix-2.8.0/doc/manual/src/language/builtin-constants.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/language/builtin-constants.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,43 @@ +# Built-in Constants + +These constants are built into the Nix language evaluator: + +- [`builtins`]{#builtins-builtins} (attribute set) + + Contains all the [built-in functions](./builtins.md) and values, in order to avoid polluting the global scope. + + Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: + + ```nix + if builtins ? getEnv then builtins.getEnv "PATH" else "" + ``` + +- [`builtins.currentSystem`]{#builtins-currentSystem} (string) + + The built-in value `currentSystem` evaluates to the Nix platform + identifier for the Nix installation on which the expression is being + evaluated, such as `"i686-linux"` or `"x86_64-darwin"`. + + Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + +- [`builtins.currentTime`]{#builtins-currentTime} (integer) + + Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation. + Repeated references to that name will re-use the initially obtained value. + + Example: + + ```console + $ nix repl + Welcome to Nix 2.15.1 Type :? for help. + + nix-repl> builtins.currentTime + 1683705525 + + nix-repl> builtins.currentTime + 1683705525 + ``` + + The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation. + + Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). diff -Nru nix-2.8.0/doc/manual/src/language/builtins-prefix.md nix-2.16.1+dfsg/doc/manual/src/language/builtins-prefix.md --- nix-2.8.0/doc/manual/src/language/builtins-prefix.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/language/builtins-prefix.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,16 @@ +# Built-in Functions + +This section lists the functions built into the Nix language evaluator. +All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. + +For convenience, some built-ins are can be accessed directly: + +- [`derivation`](#builtins-derivation) +- [`import`](#builtins-import) +- [`abort`](#builtins-abort) +- [`throw`](#builtins-throw) + +
+
derivation attrs
+

derivation is described in + its own section.

diff -Nru nix-2.8.0/doc/manual/src/language/builtins-suffix.md nix-2.16.1+dfsg/doc/manual/src/language/builtins-suffix.md --- nix-2.8.0/doc/manual/src/language/builtins-suffix.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/language/builtins-suffix.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1 @@ +
diff -Nru nix-2.8.0/doc/manual/src/language/constructs.md nix-2.16.1+dfsg/doc/manual/src/language/constructs.md --- nix-2.8.0/doc/manual/src/language/constructs.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/language/constructs.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,354 @@ +# Language Constructs + +## Recursive sets + +Recursive sets are just normal sets, but the attributes can refer to +each other. For example, + +```nix +rec { + x = y; + y = 123; +}.x +``` + +evaluates to `123`. Note that without `rec` the binding `x = y;` would +refer to the variable `y` in the surrounding scope, if one exists, and +would be invalid if no such variable exists. That is, in a normal +(non-recursive) set, attributes are not added to the lexical scope; in a +recursive set, they are. + +Recursive sets of course introduce the danger of infinite recursion. For +example, the expression + +```nix +rec { + x = y; + y = x; +}.x +``` + +will crash with an `infinite recursion encountered` error message. + +## Let-expressions + +A let-expression allows you to define local variables for an expression. +For instance, + +```nix +let + x = "foo"; + y = "bar"; +in x + y +``` + +evaluates to `"foobar"`. + +## Inheriting attributes + +When defining a set or in a let-expression it is often convenient to +copy variables from the surrounding lexical scope (e.g., when you want +to propagate attributes). This can be shortened using the `inherit` +keyword. For instance, + +```nix +let x = 123; in +{ inherit x; + y = 456; +} +``` + +is equivalent to + +```nix +let x = 123; in +{ x = x; + y = 456; +} +``` + +and both evaluate to `{ x = 123; y = 456; }`. (Note that this works +because `x` is added to the lexical scope by the `let` construct.) It is +also possible to inherit attributes from another set. For instance, in +this fragment from `all-packages.nix`, + +```nix +graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xlibs) libXaw; +}; + +xlibs = { + libX11 = ...; + libXaw = ...; + ... +} + +libpng = ...; +libjpg = ...; +... +``` + +the set used in the function call to the function defined in +`../tools/graphics/graphviz` inherits a number of variables from the +surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` +(the X Athena Widgets) from the `xlibs` (X11 client-side libraries) set. + +Summarizing the fragment + +```nix +... +inherit x y z; +inherit (src-set) a b c; +... +``` + +is equivalent to + +```nix +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... +``` + +when used while defining local variables in a let-expression or while +defining a set. + +## Functions + +Functions have the following form: + +```nix +pattern: body +``` + +The pattern specifies what the argument of the function must look like, +and binds variables in the body to (parts of) the argument. There are +three kinds of patterns: + + - If a pattern is a single identifier, then the function matches any + argument. Example: + + ```nix + let negate = x: !x; + concat = x: y: x + y; + in if negate true then concat "foo" "bar" else "" + ``` + + Note that `concat` is a function that takes one argument and returns + a function that takes another argument. This allows partial + parameterisation (i.e., only filling some of the arguments of a + function); e.g., + + ```nix + map (concat "foo") [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a + set containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + + ```nix + { x, y, z }: z + y + x + ``` + + can only be called with a set containing exactly the attributes `x`, + `y` and `z`. No other attributes are allowed. If you want to allow + additional arguments, you can use an ellipsis (`...`): + + ```nix + { x, y, z, ... }: z + y + x + ``` + + This works on any set that contains at least the three named + attributes. + + It is possible to provide *default values* for attributes, in + which case they are allowed to be missing. A default value is + specified by writing `name ? e`, where *e* is an arbitrary + expression. For example, + + ```nix + { x, y ? "foo", z ? "bar" }: z + y + x + ``` + + specifies a function that only requires an attribute named `x`, but + optionally accepts `y` and `z`. + + - An `@`-pattern provides a means of referring to the whole value + being matched: + + ```nix + args@{ x, y, z, ... }: z + y + x + args.a + ``` + + but can also be written as: + + ```nix + { x, y, z, ... } @ args: z + y + x + args.a + ``` + + Here `args` is bound to the entire argument, which is further + matched against the pattern `{ x, y, z, + ... }`. `@`-pattern makes mainly sense with an ellipsis(`...`) as + you can access attribute names as `a`, using `args.a`, which was + given as an additional attribute to the function. + + > **Warning** + > + > The `args@` expression is bound to the argument passed to the + > function which means that attributes with defaults that aren't + > explicitly specified in the function call won't cause an + > evaluation error, but won't exist in `args`. + > + > For instance + > + > ```nix + > let + > function = args@{ a ? 23, ... }: args; + > in + > function {} + > ```` + > + > will evaluate to an empty attribute set. + +Note that functions do not have names. If you want to give them a name, +you can bind them to an attribute, e.g., + +```nix +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; } +``` + +## Conditionals + +Conditionals look like this: + +```nix +if e1 then e2 else e3 +``` + +where *e1* is an expression that should evaluate to a Boolean value +(`true` or `false`). + +## Assertions + +Assertions are generally used to check that certain requirements on or +between features and dependencies hold. They look like this: + +```nix +assert e1; e2 +``` + +where *e1* is an expression that should evaluate to a Boolean value. If +it evaluates to `true`, *e2* is returned; otherwise expression +evaluation is aborted and a backtrace is printed. + +Here is a Nix expression for the Subversion package that shows how +assertions can be used:. + +```nix +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; ① +assert httpServer -> httpd != null && httpd.expat == expat; ② +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); ③ +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; ④ + ... +} +``` + +The points of interest are: + +1. This assertion states that if Subversion is to have support for + local repositories, then Berkeley DB is needed. So if the Subversion + function is called with the `localServer` argument set to `true` but + the `db4` argument set to `null`, then the evaluation fails. + + Note that `->` is the [logical + implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) + Boolean operation. + +2. This is a more subtle condition: if Subversion is built with Apache + (`httpServer`) support, then the Expat library (an XML library) used + by Subversion should be same as the one used by Apache. This is + because in this configuration Subversion code ends up being linked + with Apache code, and if the Expat libraries do not match, a build- + or runtime link error or incompatibility might occur. + +3. This assertion says that in order for Subversion to have SSL support + (so that it can access `https` URLs), an OpenSSL library must be + passed. Additionally, it says that *if* Apache support is enabled, + then Apache's OpenSSL should match Subversion's. (Note that if + Apache support is not enabled, we don't care about Apache's + OpenSSL.) + +4. The conditional here is not really related to assertions, but is + worth pointing out: it ensures that if SSL support is disabled, then + the Subversion derivation is not dependent on OpenSSL, even if a + non-`null` value was passed. This prevents an unnecessary rebuild of + Subversion if OpenSSL changes. + +## With-expressions + +A *with-expression*, + +```nix +with e1; e2 +``` + +introduces the set *e1* into the lexical scope of the expression *e2*. +For instance, + +```nix +let as = { x = "foo"; y = "bar"; }; +in with as; x + y +``` + +evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes +of `as` to the lexical scope in the expression `x + y`. The most common +use of `with` is in conjunction with the `import` function. E.g., + +```nix +with (import ./definitions.nix); ... +``` + +makes all attributes defined in the file `definitions.nix` available as +if they were defined locally in a `let`-expression. + +The bindings introduced by `with` do not shadow bindings introduced by +other means, e.g. + +```nix +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... +``` + +establishes the same scope as + +```nix +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... +``` + +## Comments + +Comments can be single-line, started with a `#` character, or +inline/multi-line, enclosed within `/* ... */`. diff -Nru nix-2.8.0/doc/manual/src/language/derivations.md nix-2.16.1+dfsg/doc/manual/src/language/derivations.md --- nix-2.8.0/doc/manual/src/language/derivations.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/language/derivations.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,161 @@ +# Derivations + +The most important built-in function is `derivation`, which is used to +describe a single derivation (a build task). It takes as input a set, +the attributes of which specify the inputs of the build. + + - There must be an attribute named [`system`]{#attr-system} whose value must be a + string specifying a Nix system type, such as `"i686-linux"` or + `"x86_64-darwin"`. (To figure out your system type, run `nix -vv + --version`.) The build can only be performed on a machine and + operating system matching the system type. (Nix can automatically + [forward builds for other + platforms](../advanced-topics/distributed-builds.md) by forwarding + them to other machines.) + + - There must be an attribute named `name` whose value must be a + string. This is used as a symbolic name for the package by + `nix-env`, and it is appended to the output paths of the derivation. + + - There must be an attribute named `builder` that identifies the + program that is executed to perform the build. It can be either a + derivation or a source (a local file reference, e.g., + `./builder.sh`). + + - Every attribute is passed as an environment variable to the builder. + Attribute values are translated to environment variables as follows: + + - Strings and numbers are just passed verbatim. + + - A *path* (e.g., `../foo/sources.tar`) causes the referenced file + to be copied to the store; its location in the store is put in + the environment variable. The idea is that all sources should + reside in the Nix store, since all inputs to a derivation should + reside in the Nix store. + + - A *derivation* causes that derivation to be built prior to the + present derivation; its default output path is put in the + environment variable. + + - Lists of the previous types are also allowed. They are simply + concatenated, separated by spaces. + + - `true` is passed as the string `1`, `false` and `null` are + passed as an empty string. + + - The optional attribute `args` specifies command-line arguments to be + passed to the builder. It should be a list. + + - The optional attribute `outputs` specifies a list of symbolic + outputs of the derivation. By default, a derivation produces a + single output path, denoted as `out`. However, derivations can + produce multiple output paths. This is useful because it allows + outputs to be downloaded or garbage-collected separately. For + instance, imagine a library package that provides a dynamic library, + header files, and documentation. A program that links against the + library doesn’t need the header files and documentation at runtime, + and it doesn’t need the documentation at build time. Thus, the + library package could specify: + + ```nix + outputs = [ "lib" "headers" "doc" ]; + ``` + + This will cause Nix to pass environment variables `lib`, `headers` + and `doc` to the builder containing the intended store paths of each + output. The builder would typically do something like + + ```bash + ./configure \ + --libdir=$lib/lib \ + --includedir=$headers/include \ + --docdir=$doc/share/doc + ``` + + for an Autoconf-style package. You can refer to each output of a + derivation by selecting it as an attribute, e.g. + + ```nix + buildInputs = [ pkg.lib pkg.headers ]; + ``` + + The first element of `outputs` determines the *default output*. + Thus, you could also write + + ```nix + buildInputs = [ pkg pkg.headers ]; + ``` + + since `pkg` is equivalent to `pkg.lib`. + +The function `mkDerivation` in the Nixpkgs standard environment is a +wrapper around `derivation` that adds a default value for `system` and +always uses Bash as the builder, to which the supplied builder is passed +as a command-line argument. See the Nixpkgs manual for details. + +The builder is executed as follows: + + - A temporary directory is created under the directory specified by + `TMPDIR` (default `/tmp`) where the build will take place. The + current directory is changed to this directory. + + - The environment is cleared and set to the derivation attributes, as + specified above. + + - In addition, the following variables are set: + + - `NIX_BUILD_TOP` contains the path of the temporary directory for + this build. + + - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the + temporary directory. This is to prevent the builder from + accidentally writing temporary files anywhere else. Doing so + might cause interference by other processes. + + - `PATH` is set to `/path-not-set` to prevent shells from + initialising it to their built-in default value. + + - `HOME` is set to `/homeless-shelter` to prevent programs from + using `/etc/passwd` or the like to find the user's home + directory, which could cause impurity. Usually, when `HOME` is + set, it is used as the location of the home directory, even if + it points to a non-existent path. + + - `NIX_STORE` is set to the path of the top-level Nix store + directory (typically, `/nix/store`). + + - For each output declared in `outputs`, the corresponding + environment variable is set to point to the intended path in the + Nix store for that output. Each output path is a concatenation + of the cryptographic hash of all build inputs, the `name` + attribute and the output name. (The output name is omitted if + it’s `out`.) + + - If an output path already exists, it is removed. Also, locks are + acquired to prevent multiple Nix instances from performing the same + build at the same time. + + - A log of the combined standard output and error is written to + `/nix/var/log/nix`. + + - The builder is executed with the arguments specified by the + attribute `args`. If it exits with exit code 0, it is considered to + have succeeded. + + - The temporary directory is removed (unless the `-K` option was + specified). + + - If the build was successful, Nix scans each output path for + references to input paths by looking for the hash parts of the input + paths. Since these are potential runtime dependencies, Nix registers + them as dependencies of the output paths. + + - After the build, Nix sets the last-modified timestamp on all files + in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to + the default group, and sets the mode of the file to 0444 or 0555 + (i.e., read-only, with execute permission enabled if the file was + originally executable). Note that possible `setuid` and `setgid` + bits are cleared. Setuid and setgid programs are not currently + supported by Nix. This is because the Nix archives used in + deployment have no concept of ownership information, and because it + makes the build result dependent on the user performing the build. diff -Nru nix-2.8.0/doc/manual/src/language/index.md nix-2.16.1+dfsg/doc/manual/src/language/index.md --- nix-2.8.0/doc/manual/src/language/index.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/language/index.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,581 @@ +# Nix Language + +The Nix language is + +- *domain-specific* + + It only exists for the Nix package manager: + to describe packages and configurations as well as their variants and compositions. + It is not intended for general purpose use. + +- *declarative* + + There is no notion of executing sequential steps. + Dependencies between operations are established only through data. + +- *pure* + + Values cannot change during computation. + Functions always produce the same output if their input does not change. + +- *functional* + + Functions are like any other value. + Functions can be assigned to names, taken as arguments, or returned by functions. + +- *lazy* + + Expressions are only evaluated when their value is needed. + +- *dynamically typed* + + Type errors are only detected when expressions are evaluated. + +# Overview + +This is an incomplete overview of language features, by example. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Example + + Description +
+ + + *Basic values* + + + + + + +
+ + `"hello world"` + + + + A string + +
+ + ``` + '' + multi + line + string + '' + ``` + + + + A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n string"`. + +
+ + `"hello ${ { a = "world"; }.a }"` + + `"1 2 ${toString 3}"` + + `"${pkgs.bash}/bin/sh"` + + + + String interpolation (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) + +
+ + `true`, `false` + + + + Booleans + +
+ + `null` + + + + Null value + +
+ + `123` + + + + An integer + +
+ + `3.141` + + + + A floating point number + +
+ + `/etc` + + + + An absolute path + +
+ + `./foo.png` + + + + A path relative to the file containing this Nix expression + +
+ + `~/.config` + + + + A home path. Evaluates to the `"/.config"`. + +
+ + `` + + + + Search path for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). + +
+ + *Compound values* + + + + + +
+ + `{ x = 1; y = 2; }` + + + + A set with attributes named `x` and `y` + +
+ + `{ foo.bar = 1; }` + + + + A nested set, equivalent to `{ foo = { bar = 1; }; }` + +
+ + `rec { x = "foo"; y = x + "bar"; }` + + + + A recursive set, equivalent to `{ x = "foo"; y = "foobar"; }` + +
+ + `[ "foo" "bar" "baz" ]` + + `[ 1 2 3 ]` + + `[ (f 1) { a = 1; b = 2; } [ "c" ] ]` + + + + Lists with three elements. + +
+ + *Operators* + + + + + +
+ + `"foo" + "bar"` + + + + String concatenation + +
+ + `1 + 2` + + + + Integer addition + +
+ + `"foo" == "f" + "oo"` + + + + Equality test (evaluates to `true`) + +
+ + `"foo" != "bar"` + + + + Inequality test (evaluates to `true`) + +
+ + `!true` + + + + Boolean negation + +
+ + `{ x = 1; y = 2; }.x` + + + + Attribute selection (evaluates to `1`) + +
+ + `{ x = 1; y = 2; }.z or 3` + + + + Attribute selection with default (evaluates to `3`) + +
+ + `{ x = 1; y = 2; } // { z = 3; }` + + + + Merge two sets (attributes in the right-hand set taking precedence) + +
+ + *Control structures* + + + + + +
+ + `if 1 + 1 == 2 then "yes!" else "no!"` + + + + Conditional expression + +
+ + `assert 1 + 1 == 2; "yes!"` + + + + Assertion check (evaluates to `"yes!"`). + +
+ + `let x = "foo"; y = "bar"; in x + y` + + + + Variable definition + +
+ + `with builtins; head [ 1 2 3 ]` + + + + Add all attributes from the given set to the scope (evaluates to `1`) + +
+ + *Functions (lambdas)* + + + + + +
+ + `x: x + 1` + + + + A function that expects an integer and returns it increased by 1 + +
+ + `x: y: x + y` + + + + Curried function, equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. + +
+ + `(x: x + 1) 100` + + + + A function call (evaluates to 101) + +
+ + `let inc = x: x + 1; in inc (inc (inc 100))` + + + + A function bound to a variable and subsequently called by name (evaluates to 103) + +
+ + `{ x, y }: x + y` + + + + A function that expects a set with required attributes `x` and `y` and concatenates them + +
+ + `{ x, y ? "bar" }: x + y` + + + + A function that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` + +
+ + `{ x, y, ... }: x + y` + + + + A function that expects a set with required attributes `x` and `y` and ignores any other attributes + +
+ + `{ x, y } @ args: x + y` + + `args @ { x, y }: x + y` + + + + A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args` + +
+ + *Built-in functions* + + + + + +
+ + `import ./foo.nix` + + + + Load and return Nix expression in given file + +
+ + `map (x: x + x) [ 1 2 3 ]` + + + + Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`) + +
diff -Nru nix-2.8.0/doc/manual/src/language/operators.md nix-2.16.1+dfsg/doc/manual/src/language/operators.md --- nix-2.8.0/doc/manual/src/language/operators.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/language/operators.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,167 @@ +# Operators + +| Name | Syntax | Associativity | Precedence | +|----------------------------------------|--------------------------------------------|---------------|------------| +| [Attribute selection] | *attrset* `.` *attrpath* \[ `or` *expr* \] | none | 1 | +| Function application | *func* *expr* | left | 2 | +| [Arithmetic negation][arithmetic] | `-` *number* | none | 3 | +| [Has attribute] | *attrset* `?` *attrpath* | none | 4 | +| List concatenation | *list* `++` *list* | right | 5 | +| [Multiplication][arithmetic] | *number* `*` *number* | left | 6 | +| [Division][arithmetic] | *number* `/` *number* | left | 6 | +| [Subtraction][arithmetic] | *number* `-` *number* | left | 7 | +| [Addition][arithmetic] | *number* `+` *number* | left | 7 | +| [String concatenation] | *string* `+` *string* | left | 7 | +| [Path concatenation] | *path* `+` *path* | left | 7 | +| [Path and string concatenation] | *path* `+` *string* | left | 7 | +| [String and path concatenation] | *string* `+` *path* | left | 7 | +| Logical negation (`NOT`) | `!` *bool* | none | 8 | +| [Update] | *attrset* `//` *attrset* | right | 9 | +| [Less than][Comparison] | *expr* `<` *expr* | none | 10 | +| [Less than or equal to][Comparison] | *expr* `<=` *expr* | none | 10 | +| [Greater than][Comparison] | *expr* `>` *expr* | none | 10 | +| [Greater than or equal to][Comparison] | *expr* `>=` *expr* | none | 10 | +| [Equality] | *expr* `==` *expr* | none | 11 | +| Inequality | *expr* `!=` *expr* | none | 11 | +| Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | +| Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | +| [Logical implication] | *bool* `->` *bool* | none | 14 | + +[string]: ./values.md#type-string +[path]: ./values.md#type-path +[number]: ./values.md#type-number +[list]: ./values.md#list +[attribute set]: ./values.md#attribute-set + +## Attribute selection + +Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. +If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. + + + +An attribute path is a dot-separated list of attribute names. +An attribute name can be an identifier or a string. + +> *attrpath* = *name* [ `.` *name* ]... \ +> *name* = *identifier* | *string* \ +> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` + +[Attribute selection]: #attribute-selection + +## Has attribute + +> *attrset* `?` *attrpath* + +Test whether [attribute set] *attrset* contains the attribute denoted by *attrpath*. +The result is a [Boolean] value. + +[Boolean]: ./values.md#type-boolean + +[Has attribute]: #has-attribute + +## Arithmetic + +Numbers are type-compatible: +Pure integer operations will always return integers, whereas any operation involving at least one floating point number return a floating point number. + +See also [Comparison] and [Equality]. + +The `+` operator is overloaded to also work on strings and paths. + +[arithmetic]: #arithmetic + +## String concatenation + +> *string* `+` *string* + +Concatenate two [string]s and merge their string contexts. + +[String concatenation]: #string-concatenation + +## Path concatenation + +> *path* `+` *path* + +Concatenate two [path]s. +The result is a path. + +[Path concatenation]: #path-concatenation + +## Path and string concatenation + +> *path* + *string* + +Concatenate *[path]* with *[string]*. +The result is a path. + +> **Note** +> +> The string must not have a string context that refers to a [store path]. + +[Path and string concatenation]: #path-and-string-concatenation + +## String and path concatenation + +> *string* + *path* + +Concatenate *[string]* with *[path]*. +The result is a string. + +> **Important** +> +> The file or directory at *path* must exist and is copied to the [store]. +> The path appears in the result as the corresponding [store path]. + +[store path]: ../glossary.md#gloss-store-path +[store]: ../glossary.md#gloss-store + +[String and path concatenation]: #string-and-path-concatenation + +## Update + +> *attrset1* // *attrset2* + +Update [attribute set] *attrset1* with names and values from *attrset2*. + +The returned attribute set will have of all the attributes in *attrset1* and *attrset2*. +If an attribute name is present in both, the attribute value from the latter is taken. + +[Update]: #update + +## Comparison + +Comparison is + +- [arithmetic] for [number]s +- lexicographic for [string]s and [path]s +- item-wise lexicographic for [list]s: + elements at the same index in both lists are compared according to their type and skipped if they are equal. + +All comparison operators are implemented in terms of `<`, and the following equivalencies hold: + +| comparison | implementation | +|--------------|-----------------------| +| *a* `<=` *b* | `! (` *b* `<` *a* `)` | +| *a* `>` *b* | *b* `<` *a* | +| *a* `>=` *b* | `! (` *a* `<` *b* `)` | + +[Comparison]: #comparison-operators + +## Equality + +- [Attribute sets][attribute set] and [list]s are compared recursively, and therefore are fully evaluated. +- Comparison of [function]s always returns `false`. +- Numbers are type-compatible, see [arithmetic] operators. +- Floating point numbers only differ up to a limited precision. + +[function]: ./constructs.md#functions + +[Equality]: #equality + +## Logical implication + +Equivalent to `!`*b1* `||` *b2*. + +[Logical implication]: #logical-implication + diff -Nru nix-2.8.0/doc/manual/src/language/string-interpolation.md nix-2.16.1+dfsg/doc/manual/src/language/string-interpolation.md --- nix-2.8.0/doc/manual/src/language/string-interpolation.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/language/string-interpolation.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,82 @@ +# String interpolation + +String interpolation is a language feature where a [string], [path], or [attribute name] can contain expressions enclosed in `${ }` (dollar-sign with curly brackets). + +Such a string is an *interpolated string*, and an expression inside is an *interpolated expression*. + +Interpolated expressions must evaluate to one of the following: + +- a [string] +- a [path] +- a [derivation] + +[string]: ./values.md#type-string +[path]: ./values.md#type-path +[attribute name]: ./values.md#attribute-set +[derivation]: ../glossary.md#gloss-derivation + +## Examples + +### String + +Rather than writing + +```nix +"--with-freetype2-library=" + freetype + "/lib" +``` + +(where `freetype` is a [derivation]), you can instead write + +```nix +"--with-freetype2-library=${freetype}/lib" +``` + +The latter is automatically translated to the former. + +A more complicated example (from the Nix expression for [Qt](http://www.trolltech.com/products/qt)): + +```nix +configureFlags = " + -system-zlib -system-libpng -system-libjpeg + ${if openglSupport then "-dlopen-opengl + -L${mesa}/lib -I${mesa}/include + -L${libXmu}/lib -I${libXmu}/include" else ""} + ${if threadSupport then "-thread" else "-no-thread"} +"; +``` + +Note that Nix expressions and strings can be arbitrarily nested; +in this case the outer string contains various interpolated expressions that themselves contain strings (e.g., `"-thread"`), some of which in turn contain interpolated expressions (e.g., `${mesa}`). + +### Path + +Rather than writing + +```nix +./. + "/" + foo + "-" + bar + ".nix" +``` + +or + +```nix +./. + "/${foo}-${bar}.nix" +``` + +you can instead write + +```nix +./${foo}-${bar}.nix +``` + +### Attribute name + +Attribute names can be created dynamically with string interpolation: + +```nix +let name = "foo"; in +{ + ${name} = "bar"; +} +``` + + { foo = "bar"; } diff -Nru nix-2.8.0/doc/manual/src/language/values.md nix-2.16.1+dfsg/doc/manual/src/language/values.md --- nix-2.8.0/doc/manual/src/language/values.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/language/values.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,255 @@ +# Data Types + +## Primitives + +- String + + *Strings* can be written in three ways. + + The most common way is to enclose the string between double quotes, + e.g., `"foo bar"`. Strings can span multiple lines. The special + characters `"` and `\` and the character sequence `${` must be + escaped by prefixing them with a backslash (`\`). Newlines, carriage + returns and tabs can be written as `\n`, `\r` and `\t`, + respectively. + + You can include the results of other expressions into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + + [string interpolation]: ./string-interpolation.md + + The second way to write string literals is as an *indented string*, + which is enclosed between pairs of *double single-quotes*, like so: + + ```nix + '' + This is the first line. + This is the second line. + This is the third line. + '' + ``` + + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as a + whole (disregarding the indentation of empty lines). For instance, + the first and second line are indented two spaces, while the third + line is indented four spaces. Thus, two spaces are stripped from + each line, so the resulting string is + + ```nix + "This is the first line.\nThis is the second line.\n This is the third line.\n" + ``` + + Note that the whitespace and newline following the opening `''` is + ignored if there is no non-whitespace text on the initial line. + + Indented strings support [string interpolation]. + + Since `${` and `''` have special meaning in indented strings, you + need a way to quote them. `$` can be escaped by prefixing it with + `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped + by prefixing it with `'`, i.e., `'''`. `$` removes any special + meaning from the following `$`. Linefeed, carriage-return and tab + characters can be written as `''\n`, `''\r`, `''\t`, and `''\` + escapes any other character. + + Indented strings are primarily useful in that they allow multi-line + string literals to follow the indentation of the enclosing Nix + expression, and that less escaping is typically necessary for + strings representing languages such as shell scripts and + configuration files because `''` is much less common than `"`. + Example: + + ```nix + stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... + } + ``` + + Finally, as a convenience, *URIs* as defined in appendix B of + [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as + is*, without quotes. For instance, the string + `"http://example.org/foo.tar.bz2"` can also be written as + `http://example.org/foo.tar.bz2`. + +- Number + + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). + + See [arithmetic] and [comparison] operators for semantics. + + [arithmetic]: ./operators.md#arithmetic + [comparison]: ./operators.md#comparison + +- Path + + *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at + least one slash to be recognised as such. For instance, `builder.sh` + is not a path: it's parsed as an expression that selects the + attribute `sh` from the variable `builder`. If the file name is + relative, i.e., if it does not begin with a slash, it is made + absolute at parse time relative to the directory of the Nix + expression that contained it. For instance, if a Nix expression in + `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path + is `/foo/xyzzy/fnord.nix`. + + If the first component of a path is a `~`, it is interpreted as if + the rest of the path were relative to the user's home directory. + e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user + whose home directory is `/home/edolstra`. + + Paths can also be specified between angle brackets, e.g. + ``. This means that the directories listed in the + environment variable `NIX_PATH` will be searched for the given file + or directory name. + + When an [interpolated string][string interpolation] evaluates to a path, the path is first copied into the Nix store and the resulting string is the [store path] of the newly created [store object]. + + [store path]: ../glossary.md#gloss-store-path + [store object]: ../glossary.md#gloss-store-object + + For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. + + Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. + For example, assume you used a file path in an interpolated string during a `nix repl` session. + Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new store path, since Nix might not re-read the file contents. + + Paths themselves, except those in angle brackets (`< >`), support [string interpolation]. + + At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. + + `a.${foo}/b.${bar}` is a syntactically valid division operation. + `./a.${foo}/b.${bar}` is a path. + +- Boolean + + *Booleans* with values `true` and `false`. + +- Null + + The null value, denoted as `null`. + +## List + +Lists are formed by enclosing a whitespace-separated list of values +between square brackets. For example, + +```nix +[ 123 ./foo.nix "abc" (f { x = y; }) ] +``` + +defines a list of four elements, the last being the result of a call to +the function `f`. Note that function calls have to be enclosed in +parentheses. If they had been omitted, e.g., + +```nix +[ 123 ./foo.nix "abc" f { x = y; } ] +``` + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + +## Attribute Set + +An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). + +Names and values are separated by an equal sign (`=`). +Each value is an arbitrary expression terminated by a semicolon (`;`). + +Attributes can appear in any order. +An attribute name may only occur once. + +Example: + +```nix +{ + x = 123; + text = "Hello"; + y = f { bla = 456; }; +} +``` + +This defines a set with attributes named `x`, `text`, `y`. + +Attributes can be selected from a set using the `.` operator. For +instance, + +```nix +{ a = "Foo"; b = "Bar"; }.a +``` + +evaluates to `"Foo"`. It is possible to provide a default value in an +attribute selection using the `or` keyword: + +```nix +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" +``` + +```nix +{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" +``` + +will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. + +You can use arbitrary double-quoted strings as attribute names: + +```nix +{ "$!@#?" = 123; }."$!@#?" +``` + +```nix +let bar = "bar"; in +{ "foo ${bar}" = 123; }."foo ${bar}" +``` + +Both will evaluate to `123`. + +Attribute names support [string interpolation]: + +```nix +let bar = "foo"; in +{ foo = 123; }.${bar} +``` + +```nix +let bar = "foo"; in +{ ${bar} = 123; }.foo +``` + +Both will evaluate to `123`. + +In the special case where an attribute name inside of a set declaration +evaluates to `null` (which is normally an error, as `null` cannot be coerced to +a string), that attribute is simply not added to the set: + +```nix +{ ${if foo then "bar" else null} = true; } +``` + +This will evaluate to `{}` if `foo` evaluates to `false`. + +A set that has a `__functor` attribute whose value is callable (i.e. is +itself a function or a set with a `__functor` attribute whose value is +callable) can be applied as if it were a function, with the set itself +passed in first , e.g., + +```nix +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +``` + +evaluates to `2`. This can be used to attach metadata to a function +without the caller needing to treat it specially, or to implement a form +of object-oriented programming, for example. diff -Nru nix-2.8.0/doc/manual/src/package-management/basic-package-mgmt.md nix-2.16.1+dfsg/doc/manual/src/package-management/basic-package-mgmt.md --- nix-2.8.0/doc/manual/src/package-management/basic-package-mgmt.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/package-management/basic-package-mgmt.md 2023-06-06 10:55:23.000000000 +0000 @@ -47,7 +47,7 @@ You can view the set of available packages in Nixpkgs: ```console -$ nix-env -qaP +$ nix-env --query --available --attr-path nixpkgs.aterm aterm-2.2 nixpkgs.bash bash-3.0 nixpkgs.binutils binutils-2.15 @@ -65,7 +65,7 @@ then you need to pass the path to your Nixpkgs tree using the `-f` flag: ```console -$ nix-env -qaPf /path/to/nixpkgs +$ nix-env --query --available --attr-path --file /path/to/nixpkgs aterm aterm-2.2 bash bash-3.0 … @@ -77,7 +77,7 @@ You can filter the packages by name: ```console -$ nix-env -qaP firefox +$ nix-env --query --available --attr-path firefox nixpkgs.firefox-esr firefox-91.3.0esr nixpkgs.firefox firefox-94.0.1 ``` @@ -85,7 +85,7 @@ and using regular expressions: ```console -$ nix-env -qaP 'firefox.*' +$ nix-env --query --available --attr-path 'firefox.*' ``` It is also possible to see the *status* of available packages, i.e., @@ -93,7 +93,7 @@ the system: ```console -$ nix-env -qaPs +$ nix-env --query --available --attr-path --status … -PS nixpkgs.bash bash-3.0 --S nixpkgs.binutils binutils-2.15 @@ -110,10 +110,10 @@ Nix knows that it can fetch a pre-built package from somewhere (typically a network server) instead of building it locally. -You can install a package using `nix-env -iA`. For instance, +You can install a package using `nix-env --install --attr `. For instance, ```console -$ nix-env -iA nixpkgs.subversion +$ nix-env --install --attr nixpkgs.subversion ``` will install the package called `subversion` from `nixpkgs` channel (which is, of course, the @@ -143,14 +143,14 @@ was used for installing: ```console -$ nix-env -e subversion +$ nix-env --uninstall subversion ``` Upgrading to a new version is just as easy. If you have a new release of Nix Packages, you can do: ```console -$ nix-env -uA nixpkgs.subversion +$ nix-env --upgrade --attr nixpkgs.subversion ``` This will *only* upgrade Subversion if there is a “newer” version in the @@ -163,15 +163,15 @@ You can also upgrade all packages for which there are newer versions: ```console -$ nix-env -u +$ nix-env --upgrade ``` Sometimes it’s useful to be able to ask what `nix-env` would do, without actually doing it. For instance, to find out what packages would be -upgraded by `nix-env -u`, you can do +upgraded by `nix-env --upgrade `, you can do ```console -$ nix-env -u --dry-run +$ nix-env --upgrade --dry-run (dry run; not doing anything) upgrading `libxslt-1.1.0' to `libxslt-1.1.10' upgrading `graphviz-1.10' to `graphviz-1.12' diff -Nru nix-2.8.0/doc/manual/src/package-management/binary-cache-substituter.md nix-2.16.1+dfsg/doc/manual/src/package-management/binary-cache-substituter.md --- nix-2.8.0/doc/manual/src/package-management/binary-cache-substituter.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/package-management/binary-cache-substituter.md 2023-06-06 10:55:23.000000000 +0000 @@ -9,7 +9,7 @@ not part of the Nix distribution, but you can install it from Nixpkgs: ```console -$ nix-env -iA nixpkgs.nix-serve +$ nix-env --install --attr nixpkgs.nix-serve ``` You can then start the server, listening for HTTP connections on @@ -32,13 +32,13 @@ Priority: 30 On the client side, you can tell Nix to use your binary cache using -`--option extra-binary-caches`, e.g.: +`--substituters`, e.g.: ```console -$ nix-env -iA nixpkgs.firefox --option extra-binary-caches http://avalon:8080/ +$ nix-env --install --attr nixpkgs.firefox --substituters http://avalon:8080/ ``` -The option `extra-binary-caches` tells Nix to use this binary cache in +The option `substituters` tells Nix to use this binary cache in addition to your default caches, such as . Thus, for any path in the closure of Firefox, Nix will first check if the path is available on the server `avalon` or another binary caches. @@ -47,4 +47,4 @@ You can also tell Nix to always use your binary cache by adding a line to the `nix.conf` configuration file like this: - binary-caches = http://avalon:8080/ https://cache.nixos.org/ + substituters = http://avalon:8080/ https://cache.nixos.org/ diff -Nru nix-2.8.0/doc/manual/src/package-management/channels.md nix-2.16.1+dfsg/doc/manual/src/package-management/channels.md --- nix-2.8.0/doc/manual/src/package-management/channels.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/package-management/channels.md 2023-06-06 10:55:23.000000000 +0000 @@ -43,7 +43,7 @@ you can then say ```console -$ nix-env -u +$ nix-env --upgrade ``` to upgrade all packages in your profile to the latest versions available diff -Nru nix-2.8.0/doc/manual/src/package-management/copy-closure.md nix-2.16.1+dfsg/doc/manual/src/package-management/copy-closure.md --- nix-2.8.0/doc/manual/src/package-management/copy-closure.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/package-management/copy-closure.md 2023-06-06 10:55:23.000000000 +0000 @@ -15,7 +15,7 @@ path (that is, the path and all its dependencies) to a file, and then unpack that file into another Nix store. For example, - $ nix-store --export $(nix-store -qR $(type -p firefox)) > firefox.closure + $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) > firefox.closure writes the closure of Firefox to a file. You can then copy this file to another machine and install the closure: @@ -27,7 +27,7 @@ command, e.g. to copy and install a closure directly to/on another machine: - $ nix-store --export $(nix-store -qR $(type -p firefox)) | bzip2 | \ + $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) | bzip2 | \ ssh alice@itchy.example.org "bunzip2 | nix-store --import" However, `nix-copy-closure` is generally more efficient because it only diff -Nru nix-2.8.0/doc/manual/src/package-management/package-management.md nix-2.16.1+dfsg/doc/manual/src/package-management/package-management.md --- nix-2.8.0/doc/manual/src/package-management/package-management.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/package-management/package-management.md 2023-06-06 10:55:23.000000000 +0000 @@ -1,5 +1,4 @@ This chapter discusses how to do package management with Nix, i.e., how to obtain, install, upgrade, and erase packages. This is the “user’s” perspective of the Nix system — people who want to *create* -packages should consult the [chapter on writing Nix -expressions](../expressions/writing-nix-expressions.md). +packages should consult the chapter on the [Nix language](../language/index.md). diff -Nru nix-2.8.0/doc/manual/src/package-management/profiles.md nix-2.16.1+dfsg/doc/manual/src/package-management/profiles.md --- nix-2.8.0/doc/manual/src/package-management/profiles.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/package-management/profiles.md 2023-06-06 10:55:23.000000000 +0000 @@ -39,7 +39,7 @@ would be what we would obtain if we had done ```console -$ nix-env -iA nixpkgs.subversion +$ nix-env --install --attr nixpkgs.subversion ``` on a set of Nix expressions that contained Subversion 1.1.2. @@ -54,7 +54,7 @@ generation 43 was created from generation 42 when we did ```console -$ nix-env -iA nixpkgs.subversion nixpkgs.firefox +$ nix-env --install --attr nixpkgs.subversion nixpkgs.firefox ``` on a set of Nix expressions that contained Firefox and a new version of @@ -127,7 +127,7 @@ (abbreviation `-p`): ```console -$ nix-env -p /nix/var/nix/profiles/other-profile -iA nixpkgs.subversion +$ nix-env --profile /nix/var/nix/profiles/other-profile --install --attr nixpkgs.subversion ``` This will *not* change the `~/.nix-profile` symlink. diff -Nru nix-2.8.0/doc/manual/src/package-management/s3-substituter.md nix-2.16.1+dfsg/doc/manual/src/package-management/s3-substituter.md --- nix-2.8.0/doc/manual/src/package-management/s3-substituter.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/package-management/s3-substituter.md 2023-06-06 10:55:23.000000000 +0000 @@ -1,41 +1,11 @@ # Serving a Nix store via S3 -Nix has built-in support for storing and fetching store paths from +Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) +for storing and fetching store paths from Amazon S3 and S3-compatible services. This uses the same *binary* cache mechanism that Nix usually uses to fetch prebuilt binaries from [cache.nixos.org](https://cache.nixos.org/). -The following options can be specified as URL parameters to the S3 URL: - - - `profile`\ - The name of the AWS configuration profile to use. By default Nix - will use the `default` profile. - - - `region`\ - The region of the S3 bucket. `us–east-1` by default. - - If your bucket is not in `us–east-1`, you should always explicitly - specify the region parameter. - - - `endpoint`\ - The URL to your S3-compatible service, for when not using Amazon S3. - Do not specify this value if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - - - `scheme`\ - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - In this example we will use the bucket named `example-nix-cache`. ## Anonymous Reads to your S3-compatible binary cache diff -Nru nix-2.8.0/doc/manual/src/package-management/ssh-substituter.md nix-2.16.1+dfsg/doc/manual/src/package-management/ssh-substituter.md --- nix-2.8.0/doc/manual/src/package-management/ssh-substituter.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/package-management/ssh-substituter.md 2023-06-06 10:55:23.000000000 +0000 @@ -6,7 +6,7 @@ available on the server `avalon`: ```console -$ nix-env -iA nixpkgs.firefox --substituters ssh://alice@avalon +$ nix-env --install --attr nixpkgs.firefox --substituters ssh://alice@avalon ``` This works similar to the binary cache substituter that Nix usually @@ -25,7 +25,7 @@ into your profile, e.g. ```console -$ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --substituters +$ nix-store --realise /nix/store/m85bxg…-firefox-34.0.5 --substituters ssh://alice@avalon ``` diff -Nru nix-2.8.0/doc/manual/src/quick-start.md nix-2.16.1+dfsg/doc/manual/src/quick-start.md --- nix-2.8.0/doc/manual/src/quick-start.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/quick-start.md 2023-06-06 10:55:23.000000000 +0000 @@ -4,22 +4,22 @@ documentation. For more in-depth information you are kindly referred to subsequent chapters. -1. Install single-user Nix by running the following: +1. Install Nix by running the following: ```console - $ bash <(curl -L https://nixos.org/nix/install) + $ curl -L https://nixos.org/nix/install | sh ``` - This will install Nix in `/nix`. The install script will create - `/nix` using `sudo`, so make sure you have sufficient rights. (For - other installation methods, see - [here](installation/installation.md).) + The install script will use `sudo`, so make sure you have sufficient rights. + On Linux, `--daemon` can be omitted for a single-user install. + + For other installation methods, see [here](installation/installation.md). 1. See what installable packages are currently available in the channel: ```console - $ nix-env -qaP + $ nix-env --query --available --attr-path nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3 nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5 nixpkgs.firefox firefox-33.0.2 @@ -31,7 +31,7 @@ 1. Install some packages from the channel: ```console - $ nix-env -iA nixpkgs.hello + $ nix-env --install --attr nixpkgs.hello ``` This should download pre-built packages; it should not build them @@ -49,13 +49,13 @@ 1. Uninstall a package: ```console - $ nix-env -e hello + $ nix-env --uninstall hello ``` 1. You can also test a package without installing it: ```console - $ nix-shell -p hello + $ nix-shell --packages hello ``` This builds or downloads GNU Hello and its dependencies, then drops @@ -76,7 +76,7 @@ ```console $ nix-channel --update nixpkgs - $ nix-env -u '*' + $ nix-env --upgrade '*' ``` The latter command will upgrade each installed package for which @@ -95,5 +95,5 @@ them: ```console - $ nix-collect-garbage -d + $ nix-collect-garbage --delete-old ``` diff -Nru nix-2.8.0/doc/manual/src/release-notes/rl-2.10.md nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.10.md --- nix-2.8.0/doc/manual/src/release-notes/rl-2.10.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.10.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,31 @@ +# Release 2.10 (2022-07-11) + +* `nix repl` now takes installables on the command line, unifying the usage + with other commands that use `--file` and `--expr`. Primary breaking change + is for the common usage of `nix repl ''` which can be recovered with + `nix repl --file ''` or `nix repl --expr 'import {}'`. + + This is currently guarded by the `repl-flake` experimental feature. + +* A new function `builtins.traceVerbose` is available. It is similar + to `builtins.trace` if the `trace-verbose` setting is set to true, + and it is a no-op otherwise. + +* `nix search` has a new flag `--exclude` to filter out packages. + +* On Linux, if `/nix` doesn't exist and cannot be created and you're + not running as root, Nix will automatically use + `~/.local/share/nix/root` as a chroot store. This enables non-root + users to download the statically linked Nix binary and have it work + out of the box, e.g. + + ``` + # ~/nix run nixpkgs#hello + warning: '/nix' does not exists, so Nix will use '/home/ubuntu/.local/share/nix/root' as a chroot store + Hello, world! + ``` + +* `flake-registry.json` is now fetched from `channels.nixos.org`. + +* Nix can now be built with LTO by passing `--enable-lto` to `configure`. + LTO is currently only supported when building with GCC. diff -Nru nix-2.8.0/doc/manual/src/release-notes/rl-2.11.md nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.11.md --- nix-2.8.0/doc/manual/src/release-notes/rl-2.11.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.11.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,5 @@ +# Release 2.11 (2022-08-24) + +* `nix copy` now copies the store paths in parallel as much as possible (again). + This doesn't apply for the `daemon` and `ssh-ng` stores which copy everything + in one batch to avoid latencies issues. diff -Nru nix-2.8.0/doc/manual/src/release-notes/rl-2.12.md nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.12.md --- nix-2.8.0/doc/manual/src/release-notes/rl-2.12.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.12.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,43 @@ +# Release 2.12 (2022-12-06) + +* On Linux, Nix can now run builds in a user namespace where they run + as root (UID 0) and have 65,536 UIDs available. + + This is primarily useful for running containers such as `systemd-nspawn` + inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. + + [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. + + A build can enable this by setting the derivation attribute: + + ``` + requiredSystemFeatures = [ "uid-range" ]; + ``` + + The `uid-range` [system feature] requires the [`auto-allocate-uids`] + setting to be enabled. + + [system feature]: ../command-ref/conf-file.md#conf-system-features + +* Nix can now automatically pick UIDs for builds, removing the need to + create `nixbld*` user accounts. See [`auto-allocate-uids`]. + + [`auto-allocate-uids`]: ../command-ref/conf-file.md#conf-auto-allocate-uids + +* On Linux, Nix has experimental support for running builds inside a + cgroup. See + [`use-cgroups`](../command-ref/conf-file.md#conf-use-cgroups). + +* `` now accepts an additional argument `impure` which + defaults to `false`. If it is set to `true`, the `hash` and `sha256` + arguments will be ignored and the resulting derivation will have + `__impure` set to `true`, making it an impure derivation. + +* If `builtins.readFile` is called on a file with context, then only + the parts of the context that appear in the content of the file are + retained. This avoids a lot of spurious errors where strings end up + having a context just because they are read from a store path + ([#7260](https://github.com/NixOS/nix/pull/7260)). + +* `nix build --json` now prints some statistics about top-level + derivations, such as CPU statistics when cgroups are enabled. diff -Nru nix-2.8.0/doc/manual/src/release-notes/rl-2.13.md nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.13.md --- nix-2.8.0/doc/manual/src/release-notes/rl-2.13.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.13.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,44 @@ +# Release 2.13 (2023-01-17) + +* The `repeat` and `enforce-determinism` options have been removed + since they had been broken under many circumstances for a long time. + +* You can now use [flake references] in the [old command line interface], e.g. + + [flake references]: ../command-ref/new-cli/nix3-flake.md#flake-references + [old command line interface]: ../command-ref/main-commands.md + + ```shell-session + # nix-build flake:nixpkgs -A hello + # nix-build -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 \ + '' -A hello + # NIX_PATH=nixpkgs=flake:nixpkgs nix-build '' -A hello + ``` + +* Instead of "antiquotation", the more common term [string interpolation](../language/string-interpolation.md) is now used consistently. + Historical release notes were not changed. + +* Error traces have been reworked to provide detailed explanations and more + accurate error locations. A short excerpt of the trace is now shown by + default when an error occurs. + +* Allow explicitly selecting outputs in a store derivation installable, just like we can do with other sorts of installables. + For example, + ```shell-session + # nix build /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^dev + ``` + now works just as + ```shell-session + # nix build nixpkgs#glibc^dev + ``` + does already. + +* On Linux, `nix develop` now sets the + [*personality*](https://man7.org/linux/man-pages/man2/personality.2.html) + for the development shell in the same way as the actual build of the + derivation. This makes shells for `i686-linux` derivations work + correctly on `x86_64-linux`. + +* You can now disable the global flake registry by setting the `flake-registry` + configuration option to an empty string. The same can be achieved at runtime with + `--flake-registry ""`. diff -Nru nix-2.8.0/doc/manual/src/release-notes/rl-2.14.md nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.14.md --- nix-2.8.0/doc/manual/src/release-notes/rl-2.14.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.14.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,22 @@ +# Release 2.14 (2023-02-28) + +* A new function `builtins.readFileType` is available. It is similar to + `builtins.readDir` but acts on a single file or directory. + +* In flakes, the `.outPath` attribute of a flake now always refers to + the directory containing the `flake.nix`. This was not the case for + when `flake.nix` was in a subdirectory of e.g. a Git repository. + The root of the source of a flake in a subdirectory is still + available in `.sourceInfo.outPath`. + +* In derivations that use structured attributes, you can now use `unsafeDiscardReferences` + to disable scanning a given output for runtime dependencies: + ```nix + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + ``` + This is useful e.g. when generating self-contained filesystem images with + their own embedded Nix store: hashes found inside such an image refer + to the embedded store and not to the host's Nix store. + + This requires the `discard-references` experimental feature. diff -Nru nix-2.8.0/doc/manual/src/release-notes/rl-2.15.md nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.15.md --- nix-2.8.0/doc/manual/src/release-notes/rl-2.15.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.15.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,58 @@ +# Release 2.15 (2023-04-11) + +* Commands which take installables on the command line can now read them from the standard input if + passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which + exceed the OS argument limit. + +* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64` + or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64` + or `--to-sri` to convert a hash to Base64 or SRI format, respectively. + + As the choice of hash formats is no longer binary, the `--base16` flag is also added + to explicitly specify the Base16 format, which is still the default. + +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. + + The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. + Using this is better and more clear than relying on the now-removed `.drv` special handling. + + For example, + ```shell-session + $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv + ``` + + now gives info about the derivation itself, while + + ```shell-session + $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^* + ``` + provides information about each of its outputs. + +* The experimental command `nix describe-stores` has been removed. + +* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md). + +* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual. + They include all common options that can be specified and common environment variables that affect these commands. + + These pages can be viewed offline with `man` using + + * `man nix-store-` and `man nix-env-` + * `nix-store --help --` and `nix-env --help --`. + +* Nix when used as a client now checks whether the store (the server) trusts the client. + (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) + This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. + + `nix store ping` and `nix doctor` now display this information. + +* The new command `nix derivation add` allows adding derivations to the store without involving the Nix language. + It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. + It uses the same JSON layout as `nix derivation show`, and is its inverse. + +* `nix show-derivation` has been renamed to `nix derivation show`. + This matches `nix derivation add`, and avoids bloating the top-level namespace. + The old name is still kept as an alias for compatibility, however. + +* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field. + This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases. diff -Nru nix-2.8.0/doc/manual/src/release-notes/rl-2.16.md nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.16.md --- nix-2.8.0/doc/manual/src/release-notes/rl-2.16.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.16.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,8 @@ +# Release 2.16 (2023-05-31) + +* Speed-up of downloads from binary caches. + The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs). + The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs). + The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs). + +* The function [`builtins.replaceStrings`](@docroot@/language/builtins.md#builtins-replaceStrings) is now lazy in the value of its second argument `to`. That is, `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`. diff -Nru nix-2.8.0/doc/manual/src/release-notes/rl-2.9.md nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.9.md --- nix-2.8.0/doc/manual/src/release-notes/rl-2.9.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-2.9.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,47 @@ +# Release 2.9 (2022-05-30) + +* Running Nix with the new `--debugger` flag will cause it to start a + repl session if an exception is thrown during evaluation, or if + `builtins.break` is called. From there you can inspect the values + of variables and evaluate Nix expressions. In debug mode, the + following new repl commands are available: + + ``` + :env Show env stack + :bt Show trace stack + :st Show current trace + :st Change to another trace in the stack + :c Go until end of program, exception, or builtins.break(). + :s Go one step + ``` + + Read more about the debugger + [here](https://www.zknotes.com/note/5970). + +* Nix now provides better integration with zsh's `run-help` + feature. It is now included in the Nix installation in the form of + an autoloadable shell function, `run-help-nix`. It picks up Nix + subcommands from the currently typed in command and directs the user + to the associated man pages. + +* `nix repl` has a new build-and-link (`:bl`) command that builds a + derivation while creating GC root symlinks. + +* The path produced by `builtins.toFile` is now allowed to be imported + or read even with restricted evaluation. Note that this will not + work with a read-only store. + +* `nix build` has a new `--print-out-paths` flag to print the + resulting output paths. This matches the default behaviour of + `nix-build`. + +* You can now specify which outputs of a derivation `nix` should + operate on using the syntax `installable^outputs`, + e.g. `nixpkgs#glibc^dev,static` or `nixpkgs#glibc^*`. By default, + `nix` will use the outputs specified by the derivation's + `meta.outputsToInstall` attribute if it exists, or all outputs + otherwise. + +* `builtins.fetchTree` (and flake inputs) can now be used to fetch + plain files over the `http(s)` and `file` protocols in addition to + directory tarballs. diff -Nru nix-2.8.0/doc/manual/src/release-notes/rl-next.md nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-next.md --- nix-2.8.0/doc/manual/src/release-notes/rl-next.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/release-notes/rl-next.md 2023-06-06 10:55:23.000000000 +0000 @@ -1 +1,2 @@ # Release X.Y (202?-??-??) + diff -Nru nix-2.8.0/doc/manual/src/SUMMARY.md.in nix-2.16.1+dfsg/doc/manual/src/SUMMARY.md.in --- nix-2.8.0/doc/manual/src/SUMMARY.md.in 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/src/SUMMARY.md.in 2023-06-06 10:55:23.000000000 +0000 @@ -15,6 +15,7 @@ - [Multi-User Mode](installation/multi-user.md) - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) + - [Uninstalling Nix](installation/uninstall.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) @@ -26,21 +27,15 @@ - [Copying Closures via SSH](package-management/copy-closure.md) - [Serving a Nix store via SSH](package-management/ssh-substituter.md) - [Serving a Nix store via S3](package-management/s3-substituter.md) -- [Writing Nix Expressions](expressions/writing-nix-expressions.md) - - [A Simple Nix Expression](expressions/simple-expression.md) - - [Expression Syntax](expressions/expression-syntax.md) - - [Build Script](expressions/build-script.md) - - [Arguments and Variables](expressions/arguments-variables.md) - - [Building and Testing](expressions/simple-building-testing.md) - - [Generic Builder Syntax](expressions/generic-builder.md) - - [Writing Nix Expressions](expressions/expression-language.md) - - [Values](expressions/language-values.md) - - [Language Constructs](expressions/language-constructs.md) - - [Operators](expressions/language-operators.md) - - [Derivations](expressions/derivations.md) - - [Advanced Attributes](expressions/advanced-attributes.md) - - [Built-in Constants](expressions/builtin-constants.md) - - [Built-in Functions](expressions/builtins.md) +- [Nix Language](language/index.md) + - [Data Types](language/values.md) + - [Language Constructs](language/constructs.md) + - [String interpolation](language/string-interpolation.md) + - [Operators](language/operators.md) + - [Derivations](language/derivations.md) + - [Advanced Attributes](language/advanced-attributes.md) + - [Built-in Constants](language/builtin-constants.md) + - [Built-in Functions](language/builtins.md) - [Advanced Topics](advanced-topics/advanced-topics.md) - [Remote Builds](advanced-topics/distributed-builds.md) - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) @@ -50,10 +45,41 @@ - [Common Options](command-ref/opt-common.md) - [Common Environment Variables](command-ref/env-common.md) - [Main Commands](command-ref/main-commands.md) - - [nix-env](command-ref/nix-env.md) - [nix-build](command-ref/nix-build.md) - [nix-shell](command-ref/nix-shell.md) - [nix-store](command-ref/nix-store.md) + - [nix-store --add-fixed](command-ref/nix-store/add-fixed.md) + - [nix-store --add](command-ref/nix-store/add.md) + - [nix-store --delete](command-ref/nix-store/delete.md) + - [nix-store --dump-db](command-ref/nix-store/dump-db.md) + - [nix-store --dump](command-ref/nix-store/dump.md) + - [nix-store --export](command-ref/nix-store/export.md) + - [nix-store --gc](command-ref/nix-store/gc.md) + - [nix-store --generate-binary-cache-key](command-ref/nix-store/generate-binary-cache-key.md) + - [nix-store --import](command-ref/nix-store/import.md) + - [nix-store --load-db](command-ref/nix-store/load-db.md) + - [nix-store --optimise](command-ref/nix-store/optimise.md) + - [nix-store --print-env](command-ref/nix-store/print-env.md) + - [nix-store --query](command-ref/nix-store/query.md) + - [nix-store --read-log](command-ref/nix-store/read-log.md) + - [nix-store --realise](command-ref/nix-store/realise.md) + - [nix-store --repair-path](command-ref/nix-store/repair-path.md) + - [nix-store --restore](command-ref/nix-store/restore.md) + - [nix-store --serve](command-ref/nix-store/serve.md) + - [nix-store --verify-path](command-ref/nix-store/verify-path.md) + - [nix-store --verify](command-ref/nix-store/verify.md) + - [nix-env](command-ref/nix-env.md) + - [nix-env --delete-generations](command-ref/nix-env/delete-generations.md) + - [nix-env --install](command-ref/nix-env/install.md) + - [nix-env --list-generations](command-ref/nix-env/list-generations.md) + - [nix-env --query](command-ref/nix-env/query.md) + - [nix-env --rollback](command-ref/nix-env/rollback.md) + - [nix-env --set-flag](command-ref/nix-env/set-flag.md) + - [nix-env --set](command-ref/nix-env/set.md) + - [nix-env --switch-generation](command-ref/nix-env/switch-generation.md) + - [nix-env --switch-profile](command-ref/nix-env/switch-profile.md) + - [nix-env --uninstall](command-ref/nix-env/uninstall.md) + - [nix-env --upgrade](command-ref/nix-env/upgrade.md) - [Utilities](command-ref/utilities.md) - [nix-channel](command-ref/nix-channel.md) - [nix-collect-garbage](command-ref/nix-collect-garbage.md) @@ -63,14 +89,29 @@ - [nix-instantiate](command-ref/nix-instantiate.md) - [nix-prefetch-url](command-ref/nix-prefetch-url.md) - [Experimental Commands](command-ref/experimental-commands.md) -@manpages@ +{{#include ./command-ref/new-cli/SUMMARY.md}} - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) + - [Profiles](command-ref/files/profiles.md) + - [manifest.nix](command-ref/files/manifest.nix.md) + - [manifest.json](command-ref/files/manifest.json.md) + - [Channels](command-ref/files/channels.md) + - [Default Nix expression](command-ref/files/default-nix-expression.md) +- [Architecture](architecture/architecture.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) + - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) + - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) + - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) + - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) + - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md) + - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) + - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md) + - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md) + - [Release 2.9 (2022-05-30)](release-notes/rl-2.9.md) - [Release 2.8 (2022-04-19)](release-notes/rl-2.8.md) - [Release 2.7 (2022-03-07)](release-notes/rl-2.7.md) - [Release 2.6 (2022-01-24)](release-notes/rl-2.6.md) diff -Nru nix-2.8.0/doc/manual/utils.nix nix-2.16.1+dfsg/doc/manual/utils.nix --- nix-2.8.0/doc/manual/utils.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/doc/manual/utils.nix 2023-06-06 10:55:23.000000000 +0000 @@ -5,6 +5,35 @@ concatStrings = concatStringsSep ""; + attrsToList = a: + map (name: { inherit name; value = a.${name}; }) (builtins.attrNames a); + + replaceStringsRec = from: to: string: + # recursively replace occurrences of `from` with `to` within `string` + # example: + # replaceStringRec "--" "-" "hello-----world" + # => "hello-world" + let + replaced = replaceStrings [ from ] [ to ] string; + in + if replaced == string then string else replaceStringsRec from to replaced; + + squash = replaceStringsRec "\n\n\n" "\n\n"; + + trim = string: + # trim trailing spaces and squash non-leading spaces + let + trimLine = line: + let + # separate leading spaces from the rest + parts = split "(^ *)" line; + spaces = head (elemAt parts 1); + rest = elemAt parts 2; + # drop trailing spaces + body = head (split " *$" rest); + in spaces + replaceStringsRec " " " " body; + in concatStringsSep "\n" (map trimLine (splitLines string)); + # FIXME: O(n^2) unique = foldl' (acc: e: if elem e acc then acc else acc ++ [ e ]) []; @@ -12,4 +41,66 @@ filterAttrs = pred: set: listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set)); + + optionalString = cond: string: if cond then string else ""; + + showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: + let + result = squash '' + - ${if useAnchors + then ''[`${name}`](#conf-${name})'' + else ''`${name}`''} + + ${indent " " body} + ''; + + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This setting is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To change this setting, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ${name} = ... + ``` + ''; + + # separate body to cleanly handle indentation + body = '' + ${description} + + ${experimentalFeatureNote} + + **Default:** ${showDefault documentDefault defaultValue} + + ${showAliases aliases} + ''; + + showDefault = documentDefault: defaultValue: + if documentDefault then + # a StringMap value type is specified as a string, but + # this shows the value type. The empty stringmap is `null` in + # JSON, but that converts to `{ }` here. + if defaultValue == "" || defaultValue == [] || isAttrs defaultValue + then "*empty*" + else if isBool defaultValue then + if defaultValue then "`true`" else "`false`" + else "`${toString defaultValue}`" + else "*machine-specific*"; + + showAliases = aliases: + optionalString (aliases != []) + "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; + + in result; + + indent = prefix: s: + concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); + + showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo)); } diff -Nru nix-2.8.0/docker.nix nix-2.16.1+dfsg/docker.nix --- nix-2.8.0/docker.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/docker.nix 2023-06-06 10:55:23.000000000 +0000 @@ -2,8 +2,13 @@ , lib ? pkgs.lib , name ? "nix" , tag ? "latest" +, bundleNixpkgs ? true , channelName ? "nixpkgs" , channelURL ? "https://nixos.org/channels/nixpkgs-unstable" +, extraPkgs ? [] +, maxLayers ? 100 +, nixConf ? {} +, flake-registry ? null }: let defaultPkgs = with pkgs; [ @@ -22,15 +27,27 @@ findutils iana-etc git - ]; + openssh + ] ++ extraPkgs; users = { root = { uid = 0; - shell = "/bin/bash"; + shell = "${pkgs.bashInteractive}/bin/bash"; home = "/root"; gid = 0; + groups = [ "root" ]; + description = "System administrator"; + }; + + nobody = { + uid = 65534; + shell = "${pkgs.shadow}/bin/nologin"; + home = "/var/empty"; + gid = 65534; + groups = [ "nobody" ]; + description = "Unprivileged account (don't use!)"; }; } // lib.listToAttrs ( @@ -52,6 +69,7 @@ groups = { root.gid = 0; nixbld.gid = 30000; + nobody.gid = 65534; }; userToPasswd = ( @@ -120,20 +138,27 @@ (lib.attrValues (lib.mapAttrs groupToGroup groups)) ); - nixConf = { + defaultNixConf = { sandbox = "false"; build-users-group = "nixbld"; - trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; + trusted-public-keys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ]; }; - nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n"; + + nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: + let + vStr = if builtins.isList v then lib.concatStringsSep " " v else v; + in + "${n} = ${vStr}") (defaultNixConf // nixConf))) + "\n"; baseSystem = let nixpkgs = pkgs.path; - channel = pkgs.runCommand "channel-nixos" { } '' + channel = pkgs.runCommand "channel-nixos" { inherit bundleNixpkgs; } '' mkdir $out - ln -s ${nixpkgs} $out/nixpkgs - echo "[]" > $out/manifest.nix + if [ "$bundleNixpkgs" ]; then + ln -s ${nixpkgs} $out/nixpkgs + echo "[]" > $out/manifest.nix + fi ''; rootEnv = pkgs.buildPackages.buildEnv { name = "root-profile-env"; @@ -165,6 +190,12 @@ cp -a ${rootEnv}/* $out/ ln -s ${manifest} $out/manifest.nix ''; + flake-registry-path = if (flake-registry == null) then + null + else if (builtins.readFileType (toString flake-registry)) == "directory" then + "${flake-registry}/flake-registry.json" + else + flake-registry; in pkgs.runCommand "base-system" { @@ -177,7 +208,7 @@ ]; allowSubstitutes = false; preferLocalBuild = true; - } '' + } ('' env set -x mkdir -p $out/etc @@ -223,12 +254,21 @@ mkdir -p $out/bin $out/usr/bin ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh - ''; + + '' + (lib.optionalString (flake-registry-path != null) '' + nixCacheDir="/root/.cache/nix" + mkdir -p $out$nixCacheDir + globalFlakeRegistryPath="$nixCacheDir/flake-registry.json" + ln -s ${flake-registry-path} $out$globalFlakeRegistryPath + mkdir -p $out/nix/var/nix/gcroots/auto + rootName=$(${pkgs.nix}/bin/nix --extra-experimental-features nix-command hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath)) + ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName + '')); in pkgs.dockerTools.buildLayeredImageWithNixDb { - inherit name tag; + inherit name tag maxLayers; contents = [ baseSystem ]; diff -Nru nix-2.8.0/flake.lock nix-2.16.1+dfsg/flake.lock --- nix-2.8.0/flake.lock 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/flake.lock 2023-06-06 10:55:23.000000000 +0000 @@ -1,5 +1,21 @@ { "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "lowdown-src": { "flake": false, "locked": { @@ -18,17 +34,18 @@ }, "nixpkgs": { "locked": { - "lastModified": 1645296114, - "narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=", + "lastModified": 1670461440, + "narHash": "sha256-jy1LB8HOMKGJEGXgzFRLDU1CBGL0/LlkolgnqIsF0D8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1", + "rev": "04a75b2eecc0acf6239acf9dd04485ff8d14f425", "type": "github" }, "original": { - "id": "nixpkgs", - "ref": "nixos-21.05-small", - "type": "indirect" + "owner": "NixOS", + "ref": "nixos-22.11-small", + "repo": "nixpkgs", + "type": "github" } }, "nixpkgs-regression": { @@ -41,13 +58,15 @@ "type": "github" }, "original": { - "id": "nixpkgs", + "owner": "NixOS", + "repo": "nixpkgs", "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", - "type": "indirect" + "type": "github" } }, "root": { "inputs": { + "flake-compat": "flake-compat", "lowdown-src": "lowdown-src", "nixpkgs": "nixpkgs", "nixpkgs-regression": "nixpkgs-regression" diff -Nru nix-2.8.0/flake.nix nix-2.16.1+dfsg/flake.nix --- nix-2.8.0/flake.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/flake.nix 2023-06-06 10:55:23.000000000 +0000 @@ -1,60 +1,68 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "nixpkgs/nixos-21.05-small"; - inputs.nixpkgs-regression.url = "nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11-small"; + inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; + inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src }: + outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat }: let + inherit (nixpkgs) lib; - version = builtins.readFile ./.version + versionSuffix; + officialRelease = true; + + version = lib.fileContents ./.version + versionSuffix; versionSuffix = if officialRelease then "" else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - officialRelease = true; - linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ]; crossSystems = [ "armv6l-linux" "armv7l-linux" ]; - stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" ]; + stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; - forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); - forAllSystemsAndStdenvs = f: forAllSystems (system: - nixpkgs.lib.listToAttrs + forAllSystems = lib.genAttrs systems; + + forAllCrossSystems = lib.genAttrs crossSystems; + + forAllStdenvs = f: + lib.listToAttrs (map - (n: - nixpkgs.lib.nameValuePair "${n}Packages" ( - f system n - )) stdenvs - ) - ); + (stdenvName: { + name = "${stdenvName}Packages"; + value = f stdenvName; + }) + stdenvs); - forAllStdenvs = stdenvs: f: nixpkgs.lib.genAttrs stdenvs (stdenv: f stdenv); # Memoize nixpkgs for different platforms for efficiency. - nixpkgsFor = - let stdenvsPackages = forAllSystemsAndStdenvs - (system: stdenv: - import nixpkgs { - inherit system; - overlays = [ - (overlayFor (p: p.${stdenv})) - ]; - } - ); - in - # Add the `stdenvPackages` at toplevel, both because these are the ones - # we want most of the time and for backwards compatibility - forAllSystems (system: stdenvsPackages.${system} // stdenvsPackages.${system}.stdenvPackages); + nixpkgsFor = forAllSystems + (system: let + make-pkgs = crossSystem: stdenv: import nixpkgs { + inherit system crossSystem; + overlays = [ + (overlayFor (p: p.${stdenv})) + ]; + }; + stdenvs = forAllStdenvs (make-pkgs null); + native = stdenvs.stdenvPackages; + in { + inherit stdenvs native; + static = native.pkgsStatic; + cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); + }); - commonDeps = pkgs: with pkgs; rec { + commonDeps = + { pkgs + , isStatic ? pkgs.stdenv.hostPlatform.isStatic + }: + with pkgs; rec { # Use "busybox-sandbox-shell" if present, # if not (legacy) fallback and hope it's sufficient. sh = pkgs.busybox-sandbox-shell or (busybox.override { @@ -85,9 +93,18 @@ lib.optionals stdenv.isLinux [ "--with-boost=${boost}/lib" "--with-sandbox-shell=${sh}/bin/busybox" + ] + ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ "LDFLAGS=-fuse-ld=gold" ]; + testConfigureFlags = [ + "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" + ]; + + internalApiDocsConfigureFlags = [ + "--enable-internal-api-docs" + ]; nativeBuildDeps = [ @@ -95,6 +112,7 @@ buildPackages.flex (lib.getBin buildPackages.lowdown-nix) buildPackages.mdbook + buildPackages.mdbook-linkcheck buildPackages.autoconf-archive buildPackages.autoreconfHook buildPackages.pkg-config @@ -102,7 +120,7 @@ # Tests buildPackages.git buildPackages.mercurial # FIXME: remove? only needed for tests - buildPackages.jq + buildPackages.jq # Also for custom mdBook preprocessor. ] ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; @@ -113,12 +131,20 @@ libarchive boost lowdown-nix - gtest ] ++ lib.optionals stdenv.isLinux [libseccomp] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; + checkDeps = [ + gtest + rapidcheck + ]; + + internalApiDocsDeps = [ + buildPackages.doxygen + ]; + awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin) (aws-sdk-cpp.override { apis = ["s3" "transfer"]; @@ -132,18 +158,14 @@ patches = (o.patches or []) ++ [ ./boehmgc-coroutine-sp-fallback.diff ]; - })) + }) + ) nlohmann_json ]; - - perlDeps = - [ perl - perlPackages.DBDSQLite - ]; }; installScriptFor = systems: - with nixpkgsFor.x86_64-linux; + with nixpkgsFor.x86_64-linux.native; runCommand "installer-script" { buildInputs = [ nix ]; } @@ -176,7 +198,7 @@ echo "file installer $out/install" >> $out/nix-support/hydra-build-products ''; - testNixVersions = pkgs: client: daemon: with commonDeps pkgs; with pkgs.lib; pkgs.stdenv.mkDerivation { + testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation { NIX_DAEMON_PACKAGE = daemon; NIX_CLIENT_PACKAGE = client; name = @@ -192,11 +214,12 @@ VERSION_SUFFIX = versionSuffix; nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ awsDeps; + buildInputs = buildDeps ++ awsDeps ++ checkDeps; propagatedBuildInputs = propagatedDeps; enableParallelBuilding = true; + configureFlags = testConfigureFlags; # otherwise configure fails dontBuild = true; doInstallCheck = true; @@ -207,8 +230,9 @@ installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES"; }; - binaryTarball = buildPackages: nix: pkgs: + binaryTarball = nix: pkgs: let + inherit (pkgs) buildPackages; inherit (pkgs) cacert; installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; }; in @@ -264,6 +288,7 @@ echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products tar cvfJ $fn \ --owner=0 --group=0 --mode=u+rw,uga+r \ + --mtime='1970-01-01' \ --absolute-names \ --hard-dereference \ --transform "s,$TMPDIR/install,$dir/install," \ @@ -287,45 +312,67 @@ # Forward from the previous stage as we don’t want it to pick the lowdown override nixUnstable = prev.nixUnstable; - nix = with final; with commonDeps pkgs; currentStdenv.mkDerivation { + nix = + with final; + with commonDeps { + inherit pkgs; + inherit (currentStdenv.hostPlatform) isStatic; + }; + let + canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform; + + sourceByRegexInverted = rxs: origSrc: final.lib.cleanSourceWith { + filter = (path: type: + let relPath = final.lib.removePrefix (toString origSrc + "/") (toString path); + in ! lib.any (re: builtins.match re relPath != null) rxs); + src = origSrc; + }; + in currentStdenv.mkDerivation (finalAttrs: { name = "nix-${version}"; inherit version; - src = self; - + src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" ] self; VERSION_SUFFIX = versionSuffix; outputs = [ "out" "dev" "doc" ]; nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ awsDeps; + buildInputs = buildDeps + # There have been issues building these dependencies + ++ lib.optionals (currentStdenv.hostPlatform == currentStdenv.buildPlatform) awsDeps + ++ lib.optionals finalAttrs.doCheck checkDeps; propagatedBuildInputs = propagatedDeps; disallowedReferences = [ boost ]; - preConfigure = + preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic) '' # Copy libboost_context so we don't get all of Boost in our closure. # https://github.com/NixOS/nixpkgs/issues/45462 mkdir -p $out/lib cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a - ${lib.optionalString currentStdenv.isLinux '' + ${lib.optionalString currentStdenv.hostPlatform.isLinux '' chmod u+w $out/lib/*.so.* patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* ''} - ${lib.optionalString currentStdenv.isDarwin '' + ${lib.optionalString currentStdenv.hostPlatform.isDarwin '' for LIB in $out/lib/*.dylib; do chmod u+w $LIB install_name_tool -id $LIB $LIB + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true done install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib ''} ''; configureFlags = configureFlags ++ - [ "--sysconfdir=/etc" ]; + [ "--sysconfdir=/etc" ] ++ + lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++ + [ (lib.enableFeature finalAttrs.doCheck "tests") ] ++ + lib.optionals finalAttrs.doCheck testConfigureFlags ++ + lib.optional (!canRunInstalled) "--disable-doc-gen"; enableParallelBuilding = true; @@ -338,6 +385,10 @@ postInstall = '' mkdir -p $doc/nix-support echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + ${lib.optionalString currentStdenv.hostPlatform.isStatic '' + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products + ''} ${lib.optionalString currentStdenv.isDarwin '' install_name_tool \ -change ${boost}/lib/libboost_context.dylib \ @@ -346,14 +397,17 @@ ''} ''; - doInstallCheck = true; + doInstallCheck = finalAttrs.doCheck; installCheckFlags = "sysconfdir=$(out)/etc"; + installCheckTarget = "installcheck"; # work around buggy detection in stdenv - separateDebugInfo = true; + separateDebugInfo = !currentStdenv.hostPlatform.isStatic; strictDeps = true; - passthru.perl-bindings = with final; currentStdenv.mkDerivation { + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation { name = "nix-perl-${version}"; src = self; @@ -375,17 +429,18 @@ ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security; - configureFlags = '' - --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} - --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix} - ''; + configureFlags = [ + "--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}" + "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}" + ]; enableParallelBuilding = true; postUnpack = "sourceRoot=$sourceRoot/perl"; - }; + }); - }; + meta.platforms = lib.platforms.unix; + }); lowdown-nix = with final; currentStdenv.mkDerivation rec { name = "lowdown-0.9.0"; @@ -405,39 +460,56 @@ }; }; - in { + nixos-lib = import (nixpkgs + "/nixos/lib") { }; + + # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests + runNixOSTestFor = system: test: nixos-lib.runTest { + imports = [ test ]; + hostPkgs = nixpkgsFor.${system}.native; + defaults = { + nixpkgs.pkgs = nixpkgsFor.${system}.native; + }; + _module.args.nixpkgs = nixpkgs; + }; + in { # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. - overlay = overlayFor (p: p.stdenv); + overlays.default = overlayFor (p: p.stdenv); hydraJobs = { # Binary package for various platforms. - build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix); + build = forAllSystems (system: self.packages.${system}.nix); + + buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); - buildStatic = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); + buildCross = forAllCrossSystems (crossSystem: + lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); - buildCross = nixpkgs.lib.genAttrs crossSystems (crossSystem: - nixpkgs.lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); + buildNoGc = forAllSystems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];})); + + buildNoTests = forAllSystems (system: + self.packages.${system}.nix.overrideAttrs (a: { + doCheck = + assert ! a?dontCheck; + false; + }) + ); # Perl bindings for various platforms. - perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings); + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of # the installation script. - binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system} nixpkgsFor.${system}.nix nixpkgsFor.${system}); + binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); - binaryTarballCross = nixpkgs.lib.genAttrs ["x86_64-linux"] (system: builtins.listToAttrs (map (crossSystem: { - name = crossSystem; - value = let - nixpkgsCross = import nixpkgs { - inherit system crossSystem; - overlays = [ self.overlay ]; - }; - in binaryTarball nixpkgsFor.${system} self.packages.${system}."nix-${crossSystem}" nixpkgsCross; - }) crossSystems)); + binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system: + forAllCrossSystems (crossSystem: + binaryTarball + self.packages.${system}."nix-${crossSystem}" + nixpkgsFor.${system}.cross.${crossSystem})); # The first half of the installation script. This is uploaded # to https://nixos.org/nix/install. It downloads the binary @@ -447,78 +519,88 @@ installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; # docker image with Nix inside - dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); + dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); # Line coverage analysis. coverage = - with nixpkgsFor.x86_64-linux; - with commonDeps pkgs; + with nixpkgsFor.x86_64-linux.native; + with commonDeps { inherit pkgs; }; releaseTools.coverageAnalysis { name = "nix-coverage-${version}"; src = self; + configureFlags = testConfigureFlags; + enableParallelBuilding = true; nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps ++ awsDeps; + buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ checkDeps; dontInstall = false; doInstallCheck = true; + installCheckTarget = "installcheck"; # work around buggy detection in stdenv lcovFilter = [ "*/boost/*" "*-tab.*" ]; - # We call `dot', and even though we just use it to - # syntax-check generated dot files, it still requires some - # fonts. So provide those. - FONTCONFIG_FILE = texFunctions.fontsConf; + hardeningDisable = ["fortify"]; + }; + + # API docs for Nix's unstable internal C++ interfaces. + internal-api-docs = + with nixpkgsFor.x86_64-linux.native; + with commonDeps { inherit pkgs; }; + + stdenv.mkDerivation { + pname = "nix-internal-api-docs"; + inherit version; + + src = self; + + configureFlags = testConfigureFlags ++ internalApiDocsConfigureFlags; + + nativeBuildInputs = nativeBuildDeps; + buildInputs = buildDeps ++ propagatedDeps + ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; + + dontBuild = true; + + installTargets = [ "internal-api-html" ]; + + postInstall = '' + mkdir -p $out/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products + ''; }; # System tests. - tests.remoteBuilds = import ./tests/remote-builds.nix { - system = "x86_64-linux"; - inherit nixpkgs; - inherit (self) overlay; - }; + tests.authorization = runNixOSTestFor "x86_64-linux" ./tests/nixos/authorization.nix; - tests.nix-copy-closure = import ./tests/nix-copy-closure.nix { - system = "x86_64-linux"; - inherit nixpkgs; - inherit (self) overlay; - }; + tests.remoteBuilds = runNixOSTestFor "x86_64-linux" ./tests/nixos/remote-builds.nix; - tests.nssPreload = (import ./tests/nss-preload.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - inherit (self) overlay; - }); + tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix; - tests.githubFlakes = (import ./tests/github-flakes.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - inherit (self) overlay; - }); + tests.nix-copy = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy.nix; - tests.sourcehutFlakes = (import ./tests/sourcehut-flakes.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - inherit (self) overlay; - }); + tests.nssPreload = runNixOSTestFor "x86_64-linux" ./tests/nixos/nss-preload.nix; + + tests.githubFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/github-flakes.nix; + + tests.sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/sourcehut-flakes.nix; + + tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix; - tests.setuid = nixpkgs.lib.genAttrs + tests.setuid = lib.genAttrs ["i686-linux" "x86_64-linux"] - (system: - import ./tests/setuid.nix rec { - inherit nixpkgs system; - inherit (self) overlay; - }); + (system: runNixOSTestFor system ./tests/nixos/setuid.nix); + # Make sure that nix-env still produces the exact same result # on a particular version of Nixpkgs. tests.evalNixpkgs = - with nixpkgsFor.x86_64-linux; + with nixpkgsFor.x86_64-linux.native; runCommand "eval-nixos" { buildInputs = [ nix ]; } '' type -p nix-env @@ -528,13 +610,19 @@ mkdir $out ''; + tests.nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/release.nix") + { pkgs = nixpkgsFor.${system}.native; } + ); + metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { - pkgs = nixpkgsFor.x86_64-linux; + pkgs = nixpkgsFor.x86_64-linux.native; nixpkgs = nixpkgs-regression; }; installTests = forAllSystems (system: - let pkgs = nixpkgsFor.${system}; in + let pkgs = nixpkgsFor.${system}.native; in pkgs.runCommand "install-tests" { againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; againstCurrentUnstable = @@ -547,62 +635,30 @@ # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; } "touch $out"); + installerTests = import ./tests/installer { + binaryTarballs = self.hydraJobs.binaryTarball; + inherit nixpkgsFor; + }; + }; checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; - } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems)) { + nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; + } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; }); - packages = forAllSystems (system: { - inherit (nixpkgsFor.${system}) nix; - } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = let - nixpkgs = nixpkgsFor.${system}.pkgsStatic; - in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation { - name = "nix-${version}"; - - src = self; - - VERSION_SUFFIX = versionSuffix; - - outputs = [ "out" "dev" "doc" ]; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps; - - configureFlags = [ "--sysconfdir=/etc" ]; - - enableParallelBuilding = true; - - makeFlags = "profiledir=$(out)/etc/profile.d"; - - doCheck = true; - - installFlags = "sysconfdir=$(out)/etc"; - - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - mkdir -p $out/nix-support - echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - ''; - - doInstallCheck = true; - installCheckFlags = "sysconfdir=$(out)/etc"; - - stripAllList = ["bin"]; - - strictDeps = true; - - hardeningDisable = [ "pie" ]; - }; + packages = forAllSystems (system: rec { + inherit (nixpkgsFor.${system}.native) nix; + default = nix; + } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { + nix-static = nixpkgsFor.${system}.static.nix; dockerImage = let - pkgs = nixpkgsFor.${system}; + pkgs = nixpkgsFor.${system}.native; image = import ./docker.nix { inherit pkgs; tag = version; }; in pkgs.runCommand @@ -614,83 +670,64 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map (crossSystem: { - name = "nix-${crossSystem}"; - value = let - nixpkgsCross = import nixpkgs { - inherit system crossSystem; - overlays = [ self.overlay ]; - }; - in with commonDeps nixpkgsCross; nixpkgsCross.stdenv.mkDerivation { - name = "nix-${version}"; - - src = self; + } // builtins.listToAttrs (map + (crossSystem: { + name = "nix-${crossSystem}"; + value = nixpkgsFor.${system}.cross.${crossSystem}.nix; + }) + crossSystems) + // builtins.listToAttrs (map + (stdenvName: { + name = "nix-${stdenvName}"; + value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix; + }) + stdenvs))); + + devShells = let + makeShell = pkgs: stdenv: + with commonDeps { inherit pkgs; }; + stdenv.mkDerivation { + name = "nix"; - VERSION_SUFFIX = versionSuffix; - - outputs = [ "out" "dev" "doc" ]; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps; - - configureFlags = [ "--sysconfdir=/etc" "--disable-doc-gen" ]; - - enableParallelBuilding = true; - - makeFlags = "profiledir=$(out)/etc/profile.d"; - - doCheck = true; - - installFlags = "sysconfdir=$(out)/etc"; - - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - mkdir -p $out/nix-support - echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - ''; - - doInstallCheck = true; - installCheckFlags = "sysconfdir=$(out)/etc"; - }; - }) crossSystems)) // (builtins.listToAttrs (map (stdenvName: - nixpkgsFor.${system}.lib.nameValuePair - "nix-${stdenvName}" - nixpkgsFor.${system}."${stdenvName}Packages".nix - ) stdenvs))); - - defaultPackage = forAllSystems (system: self.packages.${system}.nix); - - devShell = forAllSystems (system: self.devShells.${system}.stdenvPackages); - - devShells = forAllSystemsAndStdenvs (system: stdenv: - with nixpkgsFor.${system}; - with commonDeps pkgs; - - nixpkgsFor.${system}.${stdenv}.mkDerivation { - name = "nix"; - - outputs = [ "out" "dev" "doc" ]; + outputs = [ "out" "dev" "doc" ]; - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ perlDeps; + nativeBuildInputs = nativeBuildDeps + ++ (lib.optionals stdenv.cc.isClang [ pkgs.bear pkgs.clang-tools ]); - inherit configureFlags; + buildInputs = buildDeps ++ propagatedDeps + ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; - enableParallelBuilding = true; + configureFlags = configureFlags + ++ testConfigureFlags ++ internalApiDocsConfigureFlags; - installFlags = "sysconfdir=$(out)/etc"; + enableParallelBuilding = true; - shellHook = - '' - PATH=$prefix/bin:$PATH - unset PYTHONPATH - export MANPATH=$out/share/man:$MANPATH + installFlags = "sysconfdir=$(out)/etc"; - # Make bash completion work. - XDG_DATA_DIRS+=:$out/share - ''; - }); + shellHook = + '' + PATH=$prefix/bin:$PATH + unset PYTHONPATH + export MANPATH=$out/share/man:$MANPATH + # Make bash completion work. + XDG_DATA_DIRS+=:$out/share + ''; + }; + in + forAllSystems (system: + let + makeShells = prefix: pkgs: + lib.mapAttrs' + (k: v: lib.nameValuePair "${prefix}-${k}" v) + (forAllStdenvs (stdenvName: makeShell pkgs pkgs.${stdenvName})); + in + (makeShells "native" nixpkgsFor.${system}.native) // + (makeShells "static" nixpkgsFor.${system}.static) // + (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + { + default = self.devShells.${system}.native-stdenvPackages; + } + ); }; } diff -Nru nix-2.8.0/.github/CODEOWNERS nix-2.16.1+dfsg/.github/CODEOWNERS --- nix-2.8.0/.github/CODEOWNERS 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/.github/CODEOWNERS 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,18 @@ +# Pull requests concerning the listed files will automatically invite the respective maintainers as reviewers. +# This file is not used for denoting any kind of ownership, but is merely a tool for handling notifications. +# +# Merge permissions are required for maintaining an entry in this file. +# For documentation on this mechanism, see https://help.github.com/articles/about-codeowners/ + +# Default reviewers if nothing else matches +* @edolstra + +# This file +.github/CODEOWNERS @edolstra + +# Public documentation +/doc @fricklerhandwerk +*.md @fricklerhandwerk + +# Libstore layer +/src/libstore @thufschmitt diff -Nru nix-2.8.0/.github/ISSUE_TEMPLATE/bug_report.md nix-2.16.1+dfsg/.github/ISSUE_TEMPLATE/bug_report.md --- nix-2.8.0/.github/ISSUE_TEMPLATE/bug_report.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/.github/ISSUE_TEMPLATE/bug_report.md 2023-06-06 10:55:23.000000000 +0000 @@ -30,3 +30,7 @@ **Additional context** Add any other context about the problem here. + +**Priorities** + +Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff -Nru nix-2.8.0/.github/ISSUE_TEMPLATE/feature_request.md nix-2.16.1+dfsg/.github/ISSUE_TEMPLATE/feature_request.md --- nix-2.8.0/.github/ISSUE_TEMPLATE/feature_request.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/.github/ISSUE_TEMPLATE/feature_request.md 2023-06-06 10:55:23.000000000 +0000 @@ -2,7 +2,7 @@ name: Feature request about: Suggest an idea for this project title: '' -labels: improvement +labels: feature assignees: '' --- @@ -18,3 +18,7 @@ **Additional context** Add any other context or screenshots about the feature request here. + +**Priorities** + +Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff -Nru nix-2.8.0/.github/ISSUE_TEMPLATE/installer.md nix-2.16.1+dfsg/.github/ISSUE_TEMPLATE/installer.md --- nix-2.8.0/.github/ISSUE_TEMPLATE/installer.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/.github/ISSUE_TEMPLATE/installer.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,36 @@ +--- +name: Installer issue +about: Report problems with installation +title: '' +labels: installer +assignees: '' + +--- + +## Platform + + + +- [ ] Linux: +- [ ] macOS +- [ ] WSL + +## Additional information + + + +## Output + +
Output + +```log + + + +``` + +
+ +## Priorities + +Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff -Nru nix-2.8.0/.github/ISSUE_TEMPLATE/missing_documentation.md nix-2.16.1+dfsg/.github/ISSUE_TEMPLATE/missing_documentation.md --- nix-2.8.0/.github/ISSUE_TEMPLATE/missing_documentation.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/.github/ISSUE_TEMPLATE/missing_documentation.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,31 @@ +--- +name: Missing or incorrect documentation +about: Help us improve the reference manual +title: '' +labels: documentation +assignees: '' + +--- + +## Problem + + + +## Checklist + + + +- [ ] checked [latest Nix manual] \([source]) +- [ ] checked [open documentation issues and pull requests] for possible duplicates + +[latest Nix manual]: https://nixos.org/manual/nix/unstable/ +[source]: https://github.com/NixOS/nix/tree/master/doc/manual/src +[open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation + +## Proposal + + + +## Priorities + +Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff -Nru nix-2.8.0/.github/labeler.yml nix-2.16.1+dfsg/.github/labeler.yml --- nix-2.8.0/.github/labeler.yml 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/.github/labeler.yml 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,23 @@ +"documentation": + - doc/manual/* + - src/nix/**/*.md + +"store": + - src/libstore/store-api.* + - src/libstore/*-store.* + +"fetching": + - src/libfetchers/**/* + +"repl": + - src/libcmd/repl.* + - src/nix/repl.* + +"new-cli": + - src/nix/**/* + +"tests": + # Unit tests + - src/*/tests/**/* + # Functional and integration tests + - tests/**/* diff -Nru nix-2.8.0/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md nix-2.16.1+dfsg/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md --- nix-2.8.0/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,7 +0,0 @@ -**Release Notes** -Please include relevant [release notes](https://github.com/NixOS/nix/blob/master/doc/manual/src/release-notes/rl-next.md) as needed. - - -**Testing** - -If this issue is a regression or something that should block release, please consider including a test either in the [testsuite](https://github.com/NixOS/nix/tree/master/tests) or as a [hydraJob]( https://github.com/NixOS/nix/blob/master/flake.nix#L396) so that it can be part of the [automatic checks](https://hydra.nixos.org/jobset/nix/master). diff -Nru nix-2.8.0/.github/PULL_REQUEST_TEMPLATE.md nix-2.16.1+dfsg/.github/PULL_REQUEST_TEMPLATE.md --- nix-2.8.0/.github/PULL_REQUEST_TEMPLATE.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/.github/PULL_REQUEST_TEMPLATE.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,33 @@ +# Motivation + + +# Context + + + + + + + + +# Checklist for maintainers + + + +Maintainers: tick if completed or explain if not relevant + + - [ ] agreed on idea + - [ ] agreed on implementation strategy + - [ ] tests, as appropriate + - functional tests - `tests/**.sh` + - unit tests - `src/*/tests` + - integration tests - `tests/nixos/*` + - [ ] documentation in the manual + - [ ] documentation in the internal API docs + - [ ] code and comments are self-explanatory + - [ ] commit message explains why the change was made + - [ ] new feature or incompatible change: updated release notes + +# Priorities + +Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). diff -Nru nix-2.8.0/.github/stale.yml nix-2.16.1+dfsg/.github/stale.yml --- nix-2.8.0/.github/stale.yml 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/.github/stale.yml 2023-06-06 10:55:23.000000000 +0000 @@ -1,10 +1,9 @@ # Configuration for probot-stale - https://github.com/probot/stale daysUntilStale: 180 -daysUntilClose: 365 +daysUntilClose: false exemptLabels: - "critical" + - "never-stale" staleLabel: "stale" -markComment: | - I marked this as stale due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md) -closeComment: | - I closed this issue due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md) +markComment: false +closeComment: false diff -Nru nix-2.8.0/.github/workflows/backport.yml nix-2.16.1+dfsg/.github/workflows/backport.yml --- nix-2.8.0/.github/workflows/backport.yml 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/.github/workflows/backport.yml 2023-06-06 10:55:23.000000000 +0000 @@ -2,9 +2,15 @@ on: pull_request_target: types: [closed, labeled] +permissions: + contents: read jobs: backport: name: Backport Pull Request + permissions: + # for zeebe-io/backport-action + contents: write + pull-requests: write if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) runs-on: ubuntu-latest steps: @@ -15,12 +21,12 @@ fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v0.0.7 + uses: zeebe-io/backport-action@v1.3.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} github_workspace: ${{ github.workspace }} pull_description: |- - Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}. + Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}. # should be kept in sync with `uses` version: v0.0.5 diff -Nru nix-2.8.0/.github/workflows/ci.yml nix-2.16.1+dfsg/.github/workflows/ci.yml --- nix-2.8.0/.github/workflows/ci.yml 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/.github/workflows/ci.yml 2023-06-06 10:55:23.000000000 +0000 @@ -4,10 +4,12 @@ pull_request: push: +permissions: read-all + jobs: tests: - needs: [check_cachix] + needs: [check_secrets] strategy: matrix: os: [ubuntu-latest, macos-latest] @@ -17,31 +19,40 @@ - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v17 + - uses: cachix/install-nix-action@v21 + with: + # The sandbox would otherwise be disabled by default on Darwin + extra_nix_config: "sandbox = true" - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v10 - if: needs.check_cachix.outputs.secret == 'true' + - uses: cachix/cachix-action@v12 + if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - run: nix --experimental-features 'nix-command flakes' flake check -L - check_cachix: - name: Cachix secret present for installer tests + check_secrets: + permissions: + contents: none + name: Check Cachix and Docker secrets present for installer tests runs-on: ubuntu-latest outputs: - secret: ${{ steps.secret.outputs.secret }} + cachix: ${{ steps.secret.outputs.cachix }} + docker: ${{ steps.secret.outputs.docker }} steps: - - name: Check for Cachix secret + - name: Check for secrets id: secret env: _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} - run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}" + _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} + run: | + echo "::set-output name=cachix::${{ env._CACHIX_SECRETS != '' }}" + echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" installer: - needs: [tests, check_cachix] - if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' + needs: [tests, check_secrets] + if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' runs-on: ubuntu-latest outputs: installerURL: ${{ steps.prepare-installer.outputs.installerURL }} @@ -50,8 +61,10 @@ with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v17 - - uses: cachix/cachix-action@v10 + - uses: cachix/install-nix-action@v21 + with: + install_url: https://releases.nixos.org/nix/nix-2.13.3/install + - uses: cachix/cachix-action@v12 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' @@ -60,8 +73,8 @@ run: scripts/prepare-installer-for-github-actions installer_test: - needs: [installer, check_cachix] - if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' + needs: [installer, check_secrets] + if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' strategy: matrix: os: [ubuntu-latest, macos-latest] @@ -69,28 +82,40 @@ steps: - uses: actions/checkout@v3 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v17 + - uses: cachix/install-nix-action@v21 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" - - run: nix-instantiate -E 'builtins.currentTime' --eval + - run: sudo apt install fish zsh + if: matrix.os == 'ubuntu-latest' + - run: brew install fish + if: matrix.os == 'macos-latest' + - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" + - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" + - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" + - run: exec fish -c "nix-instantiate -E 'builtins.currentTime' --eval" + - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs" + - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello" docker_push_image: - needs: [check_cachix, tests] + needs: [check_secrets, tests] if: >- github.event_name == 'push' && github.ref_name == 'master' && - needs.check_cachix.outputs.secret == 'true' + needs.check_secrets.outputs.cachix == 'true' && + needs.check_secrets.outputs.docker == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v17 + - uses: cachix/install-nix-action@v21 + with: + install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - run: echo NIX_VERSION="$(nix-instantiate --eval -E '(import ./default.nix).defaultPackage.${builtins.currentSystem}.version' | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v10 - if: needs.check_cachix.outputs.secret == 'true' + - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV + - uses: cachix/cachix-action@v12 + if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' @@ -100,7 +125,7 @@ - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - run: docker tag nix:$NIX_VERSION nixos/nix:master - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff -Nru nix-2.8.0/.github/workflows/hydra_status.yml nix-2.16.1+dfsg/.github/workflows/hydra_status.yml --- nix-2.8.0/.github/workflows/hydra_status.yml 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/.github/workflows/hydra_status.yml 2023-06-06 10:55:23.000000000 +0000 @@ -1,8 +1,12 @@ name: Hydra status + +permissions: read-all + on: schedule: - cron: "12,42 * * * *" workflow_dispatch: + jobs: check_hydra_status: name: Check Hydra status diff -Nru nix-2.8.0/.github/workflows/labels.yml nix-2.16.1+dfsg/.github/workflows/labels.yml --- nix-2.8.0/.github/workflows/labels.yml 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/.github/workflows/labels.yml 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,24 @@ +name: "Label PR" + +on: + pull_request_target: + types: [edited, opened, synchronize, reopened] + +# WARNING: +# When extending this action, be aware that $GITHUB_TOKEN allows some write +# access to the GitHub API. This means that it should not evaluate user input in +# a way that allows code injection. + +permissions: + contents: read + pull-requests: write + +jobs: + labels: + runs-on: ubuntu-latest + if: github.repository_owner == 'NixOS' + steps: + - uses: actions/labeler@v4 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + sync-labels: true diff -Nru nix-2.8.0/.gitignore nix-2.16.1+dfsg/.gitignore --- nix-2.8.0/.gitignore 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/.gitignore 2023-06-06 10:55:23.000000000 +0000 @@ -19,14 +19,19 @@ /doc/manual/nix.json /doc/manual/conf-file.json /doc/manual/builtins.json +/doc/manual/xp-features.json /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md -/doc/manual/src/expressions/builtins.md +/doc/manual/src/command-ref/experimental-features-shortlist.md +/doc/manual/src/contributing/experimental-feature-descriptions.md +/doc/manual/src/language/builtins.md # /scripts/ /scripts/nix-profile.sh /scripts/nix-profile-daemon.sh +/scripts/nix-profile.fish +/scripts/nix-profile-daemon.fish # /src/libexpr/ /src/libexpr/lexer-tab.cc @@ -35,16 +40,19 @@ /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output /src/libexpr/nix.tbl +/src/libexpr/tests/libnixexpr-tests # /src/libstore/ *.gen.* -/src/libstore/tests/libstore-tests +/src/libstore/tests/libnixstore-tests # /src/libutil/ -/src/libutil/tests/libutil-tests +/src/libutil/tests/libnixutil-tests /src/nix/nix +/src/nix/doc + # /src/nix-env/ /src/nix-env/nix-env @@ -72,13 +80,15 @@ # /tests/ /tests/test-tmp -/tests/common.sh +/tests/common/vars-and-functions.sh /tests/result* /tests/restricted-innocent /tests/shell /tests/shell.drv /tests/config.nix /tests/ca/config.nix +/tests/dyn-drv/config.nix +/tests/repl-result-out # /tests/lang/ /tests/lang/*.out @@ -99,6 +109,7 @@ *.a *.o +*.o.tmp *.so *.dylib *.dll diff -Nru nix-2.8.0/local.mk nix-2.16.1+dfsg/local.mk --- nix-2.8.0/local.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,8 @@ clean-files += Makefile.config -GLOBAL_CXXFLAGS += -Wno-deprecated-declarations +GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch +# Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. +ERROR_SWITCH_ENUM = -Werror=switch-enum $(foreach i, config.h $(wildcard src/lib*/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) diff -Nru nix-2.8.0/maintainers/backporting.md nix-2.16.1+dfsg/maintainers/backporting.md --- nix-2.8.0/maintainers/backporting.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/maintainers/backporting.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,12 @@ + +# Backporting + +To [automatically backport a pull request](https://github.com/NixOS/nix/blob/master/.github/workflows/backport.yml) to a release branch once it's merged, assign it a label of the form [`backport `](https://github.com/NixOS/nix/labels?q=backport). + +Since [GitHub Actions workflows will not trigger other workflows](https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow), checks on the automatic backport need to be triggered by another actor. +This is achieved by closing and reopening the backport pull request. + +This specifically affects the [`installer_test`] check. +Note that it only runs after the other tests, so it may take a while to appear. + +[`installer_test`]: https://github.com/NixOS/nix/blob/895dfc656a21f6252ddf48df0d1f215effa04ecb/.github/workflows/ci.yml#L70-L91 diff -Nru nix-2.8.0/maintainers/README.md nix-2.16.1+dfsg/maintainers/README.md --- nix-2.8.0/maintainers/README.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/maintainers/README.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,145 @@ +# Nix maintainers team + +## Motivation + +The team's main responsibility is to set a direction for the development of Nix and ensure that the code is in good shape. + +We aim to achieve this by improving the contributor experience and attracting more maintainers – that is, by helping other people contributing to Nix and eventually taking responsibility – in order to scale the development process to match users' needs. + +### Objectives + +- It is obvious what is worthwhile to work on. +- It is easy to find the right place in the code to make a change. +- It is clear what is expected of a pull request. +- It is predictable how to get a change merged and released. + +### Tasks + +- Establish, communicate, and maintain a technical roadmap +- Improve documentation targeted at contributors + - Record architecture and design decisions + - Elaborate contribution guides and abide to them + - Define and assert quality criteria for contributions +- Maintain the issue tracker and triage pull requests +- Help contributors succeed with pull requests that address roadmap milestones +- Manage the release lifecycle +- Regularly publish reports on work done +- Engage with third parties in the interest of the project +- Ensure the required maintainer capacity for all of the above + +## Members + +- Eelco Dolstra (@edolstra) – Team lead +- Théophane Hufschmitt (@thufschmitt) +- Valentin Gagarin (@fricklerhandwerk) +- Thomas Bereknyei (@tomberek) +- Robert Hensing (@roberth) +- John Ericson (@Ericson2314) + +## Meeting protocol + +The team meets twice a week: + +- Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) + + 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) + 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min) + +- Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) + + 1. Code review on pull requests from [In review](#in-review). + 2. Other chores and tasks. + +Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw), and published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). + +## Project board protocol + +The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. + +Items on the board progress through the following states: + +### No Status + +During the discussion meeting, the team triages new items. +To be considered, issues and pull requests must have a high-level description to provide the whole team with the necessary context at a glance. + +On every meeting, at least one item from each of the following categories is inspected: + +1. [critical](https://github.com/NixOS/nix/labels/critical) +2. [security](https://github.com/NixOS/nix/labels/security) +3. [regression](https://github.com/NixOS/nix/labels/regression) +4. [bug](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Abug+sort%3Areactions-%2B1-desc) +5. [tests of existing functionality](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Atests+-label%3Afeature+sort%3Areactions-%2B1-desc) + +- [oldest pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Acreated-asc) +- [most popular pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Areactions-%2B1-desc) +- [oldest issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Acreated-asc) +- [most popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) + +Team members can also add pull requests or issues they would like the whole team to consider. +To ensure process quality and reliability, all non-trivial pull requests must be triaged before merging. + +If there is disagreement on the general idea behind an issue or pull request, it is moved to [To discuss](#to-discuss). +Otherwise, the issue or pull request in questions get the label [`idea approved`](https://github.com/NixOS/nix/labels/idea%20approved). +For issues this means that an implementation is welcome and will be prioritised for review. +For pull requests this means that: +- Unfinished work is encouraged to be continued. +- A reviewer is assigned to take responsibility for getting the pull request merged. + The item is moved to the [Assigned](#assigned) column. +- If needed, the team can decide to do a collarorative review. + Then the item is moved to the [In review](#in-review) column, and review session is scheduled. + +What constitutes a trivial pull request is up to maintainers' judgement. + +### To discuss + +Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings. + +This may be where the merit of the change itself or the implementation strategy is contested by a team member. + +As a general guideline, the order of items is determined as follows: + +- Prioritise pull requests over issues + + Contributors who took the time to implement concrete change proposals should not wait indefinitely. + +- Prioritise fixing bugs and testing over documentation, improvements or new features + + The team values stability and accessibility higher than raw functionality. + +- Interleave issues and PRs + + This way issues without attempts at a solution get a chance to get addressed. + +### In review + +Pull requests in this column are reviewed together during work meetings. +This is both for spreading implementation knowledge and for establishing common values in code reviews. + +When the overall direction is agreed upon, even when further changes are required, the pull request is assigned to one team member. + +### Assigned + +One team member is assigned to each of these pull requests. +They will communicate with the authors, and make the final approval once all remaining issues are addressed. + +If more substantive issues arise, the assignee can move the pull request back to [To discuss](#to-discuss) or [In review](#in-review) to involve the team again. + +### Flowchart + +The process is illustrated in the following diagram: + +```mermaid +flowchart TD + discuss[To discuss] + + review[To review] + + New --> |Disagreement on idea| discuss + New & discuss --> |Consensus on idea| review + + review --> |Consensus on implementation| Assigned + + Assigned --> |Implementation issues arise| review + Assigned --> |Remaining issues fixed| Merged +``` diff -Nru nix-2.8.0/maintainers/release-process.md nix-2.16.1+dfsg/maintainers/release-process.md --- nix-2.8.0/maintainers/release-process.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/maintainers/release-process.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,182 @@ +# Nix release process + +## Release artifacts + +The release process is intended to create the following for each +release: + +* A Git tag + +* Binary tarballs in https://releases.nixos.org/?prefix=nix/ + +* Docker images + +* Closures in https://cache.nixos.org + +* (Optionally) Updated `fallback-paths.nix` in Nixpkgs + +* An updated manual on https://nixos.org/manual/nix/stable/ + +## Creating a new release from the `master` branch + +* Make sure that the [Hydra `master` jobset](https://hydra.nixos.org/jobset/nix/master) succeeds. + +* In a checkout of the Nix repo, make sure you're on `master` and run + `git pull`. + +* Move the contents of `doc/manual/src/release-notes/rl-next.md` + (except the first line) to + `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is + the contents of `.version` *without* the patch level, e.g. `2.12` + rather than `2.12.0`). + +* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like + + ``` + # Release 2.12 (2022-12-06) + ``` + +* Proof-read / edit / rearrange the release notes. Breaking changes + and highlights should go to the top. + +* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in` + (*not* `SUMMARY.md`), e.g. + + ``` + - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) + ``` + +* Run + + ```console + $ git checkout -b release-notes + $ git add doc/manual/src/release-notes/rl-$VERSION.md + $ git commit -a -m 'Release notes' + $ git push --set-upstream $REMOTE release-notes + ``` + +* Create a PR for `release-notes`. + +* Wait for the PR to be merged. + +* Create a branch for the release: + + ```console + $ git checkout master + $ git pull + $ git checkout -b $VERSION-maintenance + ``` + +* Mark the release as stable: + + ```console + $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a + ``` + + This removes the link to `rl-next.md` from the manual and sets + `officialRelease = true` in `flake.nix`. + +* Push the release branch: + + ```console + $ git push --set-upstream origin $VERSION-maintenance + ``` + +* Create a jobset for the release branch on Hydra as follows: + + * Go to the jobset of the previous release + (e.g. https://hydra.nixos.org/jobset/nix/maintenance-2.11). + + * Select `Actions -> Clone this jobset`. + + * Set identifier to `maintenance-$VERSION`. + + * Set description to `$VERSION release branch`. + + * Set flake URL to `github:NixOS/nix/$VERSION-maintenance`. + + * Hit `Create jobset`. + +* Wait for the new jobset to evaluate and build. If impatient, go to + the evaluation and select `Actions -> Bump builds to front of + queue`. + +* When the jobset evaluation has succeeded building, take note of the + evaluation ID (e.g. `1780832` in + `https://hydra.nixos.org/eval/1780832`). + +* Tag the release and upload the release artifacts to + [`releases.nixos.org`](https://releases.nixos.org/) and [Docker Hub](https://hub.docker.com/): + + ```console + $ IS_LATEST=1 ./maintainers/upload-release.pl + ``` + + Note: `IS_LATEST=1` causes the `latest-release` branch to be + force-updated. This is used by the `nixos.org` website to get the + [latest Nix manual](https://nixos.org/manual/nixpkgs/unstable/). + + TODO: This script requires the right AWS credentials. Document. + + TODO: This script currently requires a + `/home/eelco/Dev/nix-pristine` and + `/home/eelco/Dev/nixpkgs-pristine`. + + TODO: trigger nixos.org netlify: https://docs.netlify.com/configure-builds/build-hooks/ + +* Prepare for the next point release by editing `.version` to + e.g. + + ```console + $ echo 2.12.1 > .version + $ git commit -a -m 'Bump version' + $ git push + ``` + + Commit and push this to the maintenance branch. + +* Bump the version of `master`: + + ```console + $ git checkout master + $ git pull + $ NEW_VERSION=2.13.0 + $ echo -n $NEW_VERSION > .version + $ git checkout -b bump-$NEW_VERSION + $ git commit -a -m 'Bump version' + $ git push --set-upstream origin bump-$NEW_VERSION + ``` + + Make a pull request and auto-merge it. + +* Create a milestone for the next release, move all unresolved issues + from the previous milestone, and close the previous milestone. Set + the date for the next milestone 6 weeks from now. + +* Create a backport label. + +* Post an [announcement on Discourse](https://discourse.nixos.org/c/announcements/8), including the contents of + `rl-$VERSION.md`. + +## Creating a point release + +* Wait for the desired evaluation of the maintenance jobset to finish + building. + +* Run + + ```console + $ IS_LATEST=1 ./maintainers/upload-release.pl + ``` + + Omit `IS_LATEST=1` when creating a point release that is not on the + most recent stable branch. This prevents `nixos.org` to going back + to an older release. + +* Bump the version number of the release branch as above (e.g. to + `2.12.2`). + +## Recovering from mistakes + +`upload-release.pl` should be idempotent. For instance a wrong `IS_LATEST` value can be fixed that way, by running the script on the actual latest release. + diff -Nru nix-2.8.0/maintainers/upload-release.pl nix-2.16.1+dfsg/maintainers/upload-release.pl --- nix-2.8.0/maintainers/upload-release.pl 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/maintainers/upload-release.pl 2023-06-06 10:55:23.000000000 +0000 @@ -15,7 +15,6 @@ my $releasesBucketName = "nix-releases"; my $channelsBucketName = "nix-channels"; -my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine"; my $TMPDIR = $ENV{'TMPDIR'} // "/tmp"; @@ -115,10 +114,6 @@ write_file("$tmpFile.sha256", $sha256_actual); - if (! -e "$tmpFile.asc") { - system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n"; - } - return $sha256_expected; } @@ -194,7 +189,7 @@ my $configuration = (); $configuration->{content_type} = "application/octet-stream"; - if ($fn =~ /.sha256|.asc|install/) { + if ($fn =~ /.sha256|install/) { # Text files $configuration->{content_type} = "text/plain"; } @@ -204,26 +199,22 @@ } } -# Update nix-fallback-paths.nix. +# Print new nix-fallback-paths.nix. if ($isLatest) { - system("cd $nixpkgsDir && git pull") == 0 or die; - sub getStorePath { my ($jobName) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'"; } - write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", + print STDERR "nixos/modules/installer/tools/nix-fallback-paths.nix:\n" . "{\n" . " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . - "}\n"); - - system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die; + "}\n"; } # Update the "latest" symlink. diff -Nru nix-2.8.0/Makefile nix-2.16.1+dfsg/Makefile --- nix-2.8.0/Makefile 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/Makefile 2023-06-06 10:55:23.000000000 +0000 @@ -2,9 +2,7 @@ mk/precompiled-headers.mk \ local.mk \ src/libutil/local.mk \ - src/libutil/tests/local.mk \ src/libstore/local.mk \ - src/libstore/tests/local.mk \ src/libfetchers/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ @@ -19,19 +17,31 @@ misc/launchd/local.mk \ misc/upstart/local.mk \ doc/manual/local.mk \ - tests/local.mk \ - tests/plugins/local.mk + doc/internal-api/local.mk -include Makefile.config +ifeq ($(tests), yes) +makefiles += \ + src/libutil/tests/local.mk \ + src/libstore/tests/local.mk \ + src/libexpr/tests/local.mk \ + tests/local.mk \ + tests/plugins/local.mk +else +makefiles += \ + mk/disable-tests.mk +endif + OPTIMIZE = 1 ifeq ($(OPTIMIZE), 1) - GLOBAL_CXXFLAGS += -O3 + GLOBAL_CXXFLAGS += -O3 $(CXXLTO) + GLOBAL_LDFLAGS += $(CXXLTO) else GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE endif include mk/lib.mk -GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 -I src +GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src diff -Nru nix-2.8.0/Makefile.config.in nix-2.16.1+dfsg/Makefile.config.in --- nix-2.8.0/Makefile.config.in 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/Makefile.config.in 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,3 @@ -HOST_OS = @host_os@ AR = @AR@ BDW_GC_LIBS = @BDW_GC_LIBS@ BOOST_LDFLAGS = @BOOST_LDFLAGS@ @@ -7,20 +6,23 @@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ +CXXLTO = @CXXLTO@ EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_S3 = @ENABLE_S3@ GTEST_LIBS = @GTEST_LIBS@ HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ +HOST_OS = @host_os@ LDFLAGS = @LDFLAGS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@ +LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@ LOWDOWN_LIBS = @LOWDOWN_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ -LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ +RAPIDCHECK_HEADERS = @RAPIDCHECK_HEADERS@ SHELL = @bash@ SODIUM_LIBS = @SODIUM_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ @@ -30,6 +32,7 @@ datarootdir = @datarootdir@ doc_generate = @doc_generate@ docdir = @docdir@ +embedded_sandbox_shell = @embedded_sandbox_shell@ exec_prefix = @exec_prefix@ includedir = @includedir@ libdir = @libdir@ @@ -43,3 +46,5 @@ storedir = @storedir@ sysconfdir = @sysconfdir@ system = @system@ +tests = @tests@ +internal_api_docs = @internal_api_docs@ diff -Nru nix-2.8.0/misc/launchd/org.nixos.nix-daemon.plist.in nix-2.16.1+dfsg/misc/launchd/org.nixos.nix-daemon.plist.in --- nix-2.8.0/misc/launchd/org.nixos.nix-daemon.plist.in 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/misc/launchd/org.nixos.nix-daemon.plist.in 2023-06-06 10:55:23.000000000 +0000 @@ -4,8 +4,6 @@ EnvironmentVariables - NIX_SSL_CERT_FILE - /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt OBJC_DISABLE_INITIALIZE_FORK_SAFETY YES @@ -28,7 +26,7 @@ SoftResourceLimits NumberOfFiles - 4096 + 1048576 diff -Nru nix-2.8.0/misc/systemd/nix-daemon.service.in nix-2.16.1+dfsg/misc/systemd/nix-daemon.service.in --- nix-2.8.0/misc/systemd/nix-daemon.service.in 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/misc/systemd/nix-daemon.service.in 2023-06-06 10:55:23.000000000 +0000 @@ -9,6 +9,7 @@ [Service] ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process +LimitNOFILE=1048576 [Install] WantedBy=multi-user.target diff -Nru nix-2.8.0/misc/zsh/completion.zsh nix-2.16.1+dfsg/misc/zsh/completion.zsh --- nix-2.8.0/misc/zsh/completion.zsh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/misc/zsh/completion.zsh 2023-06-06 10:55:23.000000000 +0000 @@ -10,14 +10,15 @@ local -a suggestions declare -a suggestions for suggestion in ${res:1}; do - # FIXME: This doesn't work properly if the suggestion word contains a `:` - # itself - suggestions+="${suggestion/ /:}" + suggestions+=("${suggestion%% *}") done + local -a args if [[ "$tpe" == filenames ]]; then - compadd -f + args+=('-f') + elif [[ "$tpe" == attrs ]]; then + args+=('-S' '') fi - _describe 'nix' suggestions + compadd -J nix "${args[@]}" -a suggestions } _nix "$@" diff -Nru nix-2.8.0/misc/zsh/local.mk nix-2.16.1+dfsg/misc/zsh/local.mk --- nix-2.8.0/misc/zsh/local.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/misc/zsh/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -1 +1,2 @@ $(eval $(call install-file-as, $(d)/completion.zsh, $(datarootdir)/zsh/site-functions/_nix, 0644)) +$(eval $(call install-file-as, $(d)/run-help-nix, $(datarootdir)/zsh/site-functions/run-help-nix, 0644)) diff -Nru nix-2.8.0/misc/zsh/run-help-nix nix-2.16.1+dfsg/misc/zsh/run-help-nix --- nix-2.8.0/misc/zsh/run-help-nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/misc/zsh/run-help-nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,42 @@ +emulate -L zsh + +# run-help is a zsh widget that can be bound to a key. It mainly looks up the +# man page for the currently typed in command. +# +# Although run-help works for any command without requiring special support, +# it can only deduce the right man page based solely on the name of the +# command. Programs like Nix provide better integration with run-help by +# helping zsh identify Nix subcommands and their corresponding man pages. This +# is what this function does. +# +# To actually use run-help on zsh, place the following lines in your .zshrc: +# +# (( $+aliases[run-help] )) && unalias run-help +# autoload -Uz run-help run-help-nix +# +# Then also assign run-help to any key of choice: +# +# bindkey '^[h' run-help + +while [[ "$#" != 0 && "$1" == -* ]]; do + shift +done + +local -a subcommands; subcommands=( nix3 ) + +local arg +for arg in "$@"; do + if man -w "${(j:-:)subcommands}-$arg" >/dev/null 2>&1; then + subcommands+="$arg" + else + break + fi +done + +if (( $#subcommands > 1 )); then + man "${(j:-:)subcommands}" +else + man nix +fi + +return $? diff -Nru nix-2.8.0/mk/common-test.sh nix-2.16.1+dfsg/mk/common-test.sh --- nix-2.8.0/mk/common-test.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/mk/common-test.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,11 @@ +TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=') + +: ${BASH:=/usr/bin/env bash} + +init_test () { + cd tests && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null +} + +run_test_proper () { + cd $(dirname $test) && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test) +} diff -Nru nix-2.8.0/mk/cxx-big-literal.mk nix-2.16.1+dfsg/mk/cxx-big-literal.mk --- nix-2.8.0/mk/cxx-big-literal.mk 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/mk/cxx-big-literal.mk 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,5 @@ +%.gen.hh: % + @echo 'R"foo(' >> $@.tmp + $(trace-gen) cat $< >> $@.tmp + @echo ')foo"' >> $@.tmp + @mv $@.tmp $@ diff -Nru nix-2.8.0/mk/debug-test.sh nix-2.16.1+dfsg/mk/debug-test.sh --- nix-2.8.0/mk/debug-test.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/mk/debug-test.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +test=$1 + +dir="$(dirname "${BASH_SOURCE[0]}")" +source "$dir/common-test.sh" + +(init_test) +run_test_proper diff -Nru nix-2.8.0/mk/disable-tests.mk nix-2.16.1+dfsg/mk/disable-tests.mk --- nix-2.8.0/mk/disable-tests.mk 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/mk/disable-tests.mk 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,12 @@ +# This file is only active for `./configure --disable-tests`. +# Running `make check` or `make installcheck` would indicate a mistake in the +# caller. + +installcheck: + @echo "Tests are disabled. Configure without '--disable-tests', or avoid calling 'make installcheck'." + @exit 1 + +# This currently has little effect. +check: + @echo "Tests are disabled. Configure without '--disable-tests', or avoid calling 'make check'." + @exit 1 diff -Nru nix-2.8.0/mk/lib.mk nix-2.16.1+dfsg/mk/lib.mk --- nix-2.8.0/mk/lib.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/mk/lib.mk 2023-06-06 10:55:23.000000000 +0000 @@ -101,6 +101,7 @@ include mk/programs.mk include mk/patterns.mk include mk/templates.mk +include mk/cxx-big-literal.mk include mk/tests.mk diff -Nru nix-2.8.0/mk/libraries.mk nix-2.16.1+dfsg/mk/libraries.mk --- nix-2.8.0/mk/libraries.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/mk/libraries.mk 2023-06-06 10:55:23.000000000 +0000 @@ -67,6 +67,7 @@ $(1)_LDFLAGS_USE := $(1)_LDFLAGS_USE_INSTALLED := + $(1)_LIB_CLOSURE := $(1) $$(eval $$(call create-dir, $$(_d))) @@ -91,7 +92,7 @@ $(1)_PATH := $$(_d)/$$($(1)_NAME).$(SO_EXT) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) + +$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) ifndef HOST_DARWIN $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d)) @@ -105,7 +106,7 @@ $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) $$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - $$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + +$$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) ifndef HOST_DARWIN @@ -125,13 +126,15 @@ $(1)_PATH := $$(_d)/$$($(1)_NAME).a $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/ - $$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$? + $$(trace-ld) $(LD) $$(ifndef $(HOST_DARWIN),-U) -r -o $$(_d)/$$($(1)_NAME).o $$^ $$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o - $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) + $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $(1)_INSTALL_PATH := $$(libdir)/$$($(1)_NAME).a + $(1)_LIB_CLOSURE += $$($(1)_LIBS) + endif $(1)_LDFLAGS_USE += $$($(1)_LDFLAGS_PROPAGATED) diff -Nru nix-2.8.0/mk/patterns.mk nix-2.16.1+dfsg/mk/patterns.mk --- nix-2.8.0/mk/patterns.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/mk/patterns.mk 2023-06-06 10:55:23.000000000 +0000 @@ -1,10 +1,10 @@ $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" diff -Nru nix-2.8.0/mk/programs.mk nix-2.16.1+dfsg/mk/programs.mk --- nix-2.8.0/mk/programs.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/mk/programs.mk 2023-06-06 10:55:23.000000000 +0000 @@ -3,6 +3,9 @@ # Build a program with symbolic name $(1). The program is defined by # various variables prefixed by ‘$(1)_’: # +# - $(1)_NAME: the name of the program (e.g. ‘foo’); defaults to +# $(1). +# # - $(1)_DIR: the directory where the (non-installed) program will be # placed. # @@ -23,22 +26,23 @@ # - $(1)_INSTALL_DIR: the directory where the program will be # installed; defaults to $(bindir). define build-program + $(1)_NAME ?= $(1) _d := $(buildprefix)$$($(1)_DIR) _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) - _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) - $(1)_PATH := $$(_d)/$(1) + _libs := $$(foreach lib, $$($(1)_LIBS), $$(foreach lib2, $$($$(lib)_LIB_CLOSURE), $$($$(lib2)_PATH))) + $(1)_PATH := $$(_d)/$$($(1)_NAME) $$(eval $$(call create-dir, $$(_d))) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) + +$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $(1)_INSTALL_DIR ?= $$(bindir) ifdef $(1)_INSTALL_DIR - $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) + $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME) $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) @@ -49,12 +53,12 @@ _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + +$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) else $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$< + +$$(trace-install) install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$< endif endif diff -Nru nix-2.8.0/mk/run_test.sh nix-2.16.1+dfsg/mk/run_test.sh --- nix-2.8.0/mk/run_test.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/mk/run_test.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,46 +0,0 @@ -#!/bin/sh - -set -u - -red="" -green="" -yellow="" -normal="" - -post_run_msg="ran test $1..." -if [ -t 1 ]; then - red="" - green="" - yellow="" - normal="" -fi - -run_test () { - (cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null) - log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)" - status=$? -} - -run_test "$1" - -# Hack: Retry the test if it fails with “unexpected EOF reading a line” as these -# appear randomly without anyone knowing why. -# See https://github.com/NixOS/nix/issues/3605 for more info -if [[ $status -ne 0 && $status -ne 99 && \ - "$(uname)" == "Darwin" && \ - "$log" =~ "unexpected EOF reading a line" \ -]]; then - echo "$post_run_msg [${yellow}FAIL$normal] (possibly flaky, so will be retried)" - echo "$log" | sed 's/^/ /' - run_test "$1" -fi - -if [ $status -eq 0 ]; then - echo "$post_run_msg [${green}PASS$normal]" -elif [ $status -eq 99 ]; then - echo "$post_run_msg [${yellow}SKIP$normal]" -else - echo "$post_run_msg [${red}FAIL$normal]" - echo "$log" | sed 's/^/ /' - exit "$status" -fi diff -Nru nix-2.8.0/mk/run-test.sh nix-2.16.1+dfsg/mk/run-test.sh --- nix-2.8.0/mk/run-test.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/mk/run-test.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +red="" +green="" +yellow="" +normal="" + +test=$1 + +dir="$(dirname "${BASH_SOURCE[0]}")" +source "$dir/common-test.sh" + +post_run_msg="ran test $test..." +if [ -t 1 ]; then + red="" + green="" + yellow="" + normal="" +fi + +run_test () { + (init_test 2>/dev/null > /dev/null) + log="$(run_test_proper 2>&1)" && status=0 || status=$? +} + +run_test + +if [ $status -eq 0 ]; then + echo "$post_run_msg [${green}PASS$normal]" +elif [ $status -eq 99 ]; then + echo "$post_run_msg [${yellow}SKIP$normal]" +else + echo "$post_run_msg [${red}FAIL$normal]" + echo "$log" | sed 's/^/ /' + exit "$status" +fi diff -Nru nix-2.8.0/mk/tests.mk nix-2.16.1+dfsg/mk/tests.mk --- nix-2.8.0/mk/tests.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/mk/tests.mk 2023-06-06 10:55:23.000000000 +0000 @@ -8,7 +8,11 @@ .PHONY: $1.test $1.test: $1 $(test-deps) - @env TEST_NAME=$(basename $1) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null + @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null + + .PHONY: $1.test-debug + $1.test-debug: $1 $(test-deps) + @env BASH=$(bash) $(bash) mk/debug-test.sh $1 < /dev/null endef diff -Nru nix-2.8.0/perl/lib/Nix/Store.xs nix-2.16.1+dfsg/perl/lib/Nix/Store.xs --- nix-2.8.0/perl/lib/Nix/Store.xs 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/perl/lib/Nix/Store.xs 2023-06-06 10:55:23.000000000 +0000 @@ -26,8 +26,7 @@ static std::shared_ptr _store; if (!_store) { try { - loadConfFile(); - settings.lockCPU = false; + initLibStore(); _store = openStore(); } catch (Error & e) { croak("%s", e.what()); @@ -294,7 +293,13 @@ try { auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(method, h, name); + auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + .hash = { + .method = method, + .hash = h, + }, + .references = {}, + }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff -Nru nix-2.8.0/perl/Makefile nix-2.16.1+dfsg/perl/Makefile --- nix-2.8.0/perl/Makefile 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/perl/Makefile 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,6 @@ makefiles = local.mk -GLOBAL_CXXFLAGS += -g -Wall -std=c++17 -I ../src +GLOBAL_CXXFLAGS += -g -Wall -std=c++2a -I ../src -include Makefile.config diff -Nru nix-2.8.0/README.md nix-2.16.1+dfsg/README.md --- nix-2.8.0/README.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/README.md 2023-06-06 10:55:23.000000000 +0000 @@ -20,8 +20,8 @@ ## Building And Developing -See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/contributing/hacking.html) in our manual for instruction on how to -build nix from source with nix-build or how to get a development environment. +See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to +to set up a development environment and build Nix from source. ## Additional Resources diff -Nru nix-2.8.0/scripts/install.in nix-2.16.1+dfsg/scripts/install.in --- nix-2.8.0/scripts/install.in 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/scripts/install.in 2023-06-06 10:55:23.000000000 +0000 @@ -40,12 +40,12 @@ path=@tarballPath_aarch64-linux@ system=aarch64-linux ;; - Linux.armv6l_linux) + Linux.armv6l) hash=@tarballHash_armv6l-linux@ path=@tarballPath_armv6l-linux@ system=armv6l-linux ;; - Linux.armv7l_linux) + Linux.armv7l) hash=@tarballHash_armv7l-linux@ path=@tarballPath_armv7l-linux@ system=armv7l-linux diff -Nru nix-2.8.0/scripts/install-systemd-multi-user.sh nix-2.16.1+dfsg/scripts/install-systemd-multi-user.sh --- nix-2.8.0/scripts/install-systemd-multi-user.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/scripts/install-systemd-multi-user.sh 2023-06-06 10:55:23.000000000 +0000 @@ -24,12 +24,17 @@ EOF } +escape_systemd_env() { + temp_var="${1//\'/\\\'}" + echo "${temp_var//\%/%%}" +} + # Gather all non-empty proxy environment variables into a string create_systemd_proxy_env() { vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY" for v in $vars; do if [ "x${!v:-}" != "x" ]; then - echo "Environment=${v}=${!v}" + echo "Environment=${v}=$(escape_systemd_env ${!v})" fi done } @@ -87,7 +92,7 @@ task "Setting up the nix-daemon systemd service" _sudo "to create the nix-daemon tmpfiles config" \ - ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST + ln -sfn "/nix/var/nix/profiles/default$TMPFILES_SRC" "$TMPFILES_DEST" _sudo "to run systemd-tmpfiles once to pick that path up" \ systemd-tmpfiles --create --prefix=/nix/var/nix diff -Nru nix-2.8.0/scripts/local.mk nix-2.16.1+dfsg/scripts/local.mk --- nix-2.8.0/scripts/local.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/scripts/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -6,6 +6,8 @@ profiledir = $(sysconfdir)/profile.d $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) +$(eval $(call install-file-as, $(d)/nix-profile.fish, $(profiledir)/nix.fish, 0644)) $(eval $(call install-file-as, $(d)/nix-profile-daemon.sh, $(profiledir)/nix-daemon.sh, 0644)) +$(eval $(call install-file-as, $(d)/nix-profile-daemon.fish, $(profiledir)/nix-daemon.fish, 0644)) clean-files += $(nix_noinst_scripts) diff -Nru nix-2.8.0/scripts/nix-profile-daemon.fish.in nix-2.16.1+dfsg/scripts/nix-profile-daemon.fish.in --- nix-2.8.0/scripts/nix-profile-daemon.fish.in 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/scripts/nix-profile-daemon.fish.in 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,49 @@ +function add_path --argument-names new_path + if type -q fish_add_path + # fish 3.2.0 or newer + fish_add_path --prepend --global $new_path + else + # older versions of fish + if not contains $new_path $fish_user_paths + set --global fish_user_paths $new_path $fish_user_paths + end + end +end + +# Only execute this file once per shell. +if test -n "$__ETC_PROFILE_NIX_SOURCED" + exit +end + +set __ETC_PROFILE_NIX_SOURCED 1 + +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" + +# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. +if test -n "$NIX_SSH_CERT_FILE" + : # Allow users to override the NIX_SSL_CERT_FILE +else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch + set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt +else if test -e /etc/ssl/ca-bundle.pem # openSUSE Tumbleweed + set --export NIX_SSL_CERT_FILE /etc/ssl/ca-bundle.pem +else if test -e /etc/ssl/certs/ca-bundle.crt # Old NixOS + set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt +else if test -e /etc/pki/tls/certs/ca-bundle.crt # Fedora, CentOS + set --export NIX_SSL_CERT_FILE /etc/pki/tls/certs/ca-bundle.crt +else if test -e "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" # fall back to cacert in Nix profile + set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" +else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile + set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" +else + # Fall back to what is in the nix profiles, favouring whatever is defined last. + for i in $NIX_PROFILES + if test -e "$i/etc/ssl/certs/ca-bundle.crt" + set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" + end + end +end + +add_path "@localstatedir@/nix/profiles/default/bin" +add_path "$HOME/.nix-profile/bin" + +functions -e add_path diff -Nru nix-2.8.0/scripts/nix-profile-daemon.sh.in nix-2.16.1+dfsg/scripts/nix-profile-daemon.sh.in --- nix-2.8.0/scripts/nix-profile-daemon.sh.in 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/scripts/nix-profile-daemon.sh.in 2023-06-06 10:55:23.000000000 +0000 @@ -2,7 +2,33 @@ if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 -export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" +NIX_LINK=$HOME/.nix-profile +if [ -n "${XDG_STATE_HOME-}" ]; then + NIX_LINK_NEW="$XDG_STATE_HOME/nix/profile" +else + NIX_LINK_NEW=$HOME/.local/state/nix/profile +fi +if [ -e "$NIX_LINK_NEW" ]; then + NIX_LINK="$NIX_LINK_NEW" +else + if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + warning="\033[1;35mwarning:\033[0m" + printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then + printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 + else + # This should be an exceptionally rare occasion: the only way to get it would be to + # 1. Update to newer Nix; + # 2. Remove .nix-profile; + # 3. Set the $NIX_LINK_NEW to something other than the default user profile; + # 4. Roll back to older Nix. + # If someone did all that, they can probably figure out how to migrate the profile. + printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 + fi + fi +fi + +export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then @@ -34,4 +60,5 @@ unset -f check_nix_profiles fi -export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH" +export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH" +unset NIX_LINK diff -Nru nix-2.8.0/scripts/nix-profile.fish.in nix-2.16.1+dfsg/scripts/nix-profile.fish.in --- nix-2.8.0/scripts/nix-profile.fish.in 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/scripts/nix-profile.fish.in 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,51 @@ +function add_path --argument-names new_path + if type -q fish_add_path + # fish 3.2.0 or newer + fish_add_path --prepend --global $new_path + else + # older versions of fish + if not contains $new_path $fish_user_paths + set --global fish_user_paths $new_path $fish_user_paths + end + end +end + +if test -n "$HOME" && test -n "$USER" + + # Set up the per-user profile. + + set NIX_LINK $HOME/.nix-profile + + # Set up environment. + # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix + set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" + + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. + if test -n "$NIX_SSH_CERT_FILE" + : # Allow users to override the NIX_SSL_CERT_FILE + else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch + set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt + else if test -e /etc/ssl/ca-bundle.pem # openSUSE Tumbleweed + set --export NIX_SSL_CERT_FILE /etc/ssl/ca-bundle.pem + else if test -e /etc/ssl/certs/ca-bundle.crt # Old NixOS + set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt + else if test -e /etc/pki/tls/certs/ca-bundle.crt # Fedora, CentOS + set --export NIX_SSL_CERT_FILE /etc/pki/tls/certs/ca-bundle.crt + else if test -e "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" # fall back to cacert in Nix profile + set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" + else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile + set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" + end + + # Only use MANPATH if it is already set. In general `man` will just simply + # pick up `.nix-profile/share/man` because is it close to `.nix-profile/bin` + # which is in the $PATH. For more info, run `manpath -d`. + if set --query MANPATH + set --export --prepend --path MANPATH "$NIX_LINK/share/man" + end + + add_path "$NIX_LINK/bin" + set --erase NIX_LINK +end + +functions -e add_path diff -Nru nix-2.8.0/scripts/nix-profile.sh.in nix-2.16.1+dfsg/scripts/nix-profile.sh.in --- nix-2.8.0/scripts/nix-profile.sh.in 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/scripts/nix-profile.sh.in 2023-06-06 10:55:23.000000000 +0000 @@ -1,13 +1,36 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up the per-user profile. - # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix - NIX_LINK=$HOME/.nix-profile + NIX_LINK="$HOME/.nix-profile" + if [ -n "${XDG_STATE_HOME-}" ]; then + NIX_LINK_NEW="$XDG_STATE_HOME/nix/profile" + else + NIX_LINK_NEW="$HOME/.local/state/nix/profile" + fi + if [ -e "$NIX_LINK_NEW" ]; then + NIX_LINK="$NIX_LINK_NEW" + else + if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + warning="\033[1;35mwarning:\033[0m" + printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then + printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 + else + # This should be an exceptionally rare occasion: the only way to get it would be to + # 1. Update to newer Nix; + # 2. Remove .nix-profile; + # 3. Set the $NIX_LINK_NEW to something other than the default user profile; + # 4. Roll back to older Nix. + # If someone did all that, they can probably figure out how to migrate the profile. + printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 + fi + fi + fi # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix - export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" + export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch @@ -32,5 +55,5 @@ fi export PATH="$NIX_LINK/bin:$PATH" - unset NIX_LINK + unset NIX_LINK NIX_LINK_NEW fi diff -Nru nix-2.8.0/src/build-remote/build-remote.cc nix-2.16.1+dfsg/src/build-remote/build-remote.cc --- nix-2.8.0/src/build-remote/build-remote.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/build-remote/build-remote.cc 2023-06-06 10:55:23.000000000 +0000 @@ -72,6 +72,7 @@ settings.set(name, value); } + auto maxBuildJobs = settings.maxBuildJobs; settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work initPlugins(); @@ -112,10 +113,14 @@ drvPath = store->parseStorePath(readString(source)); auto requiredFeatures = readStrings>(source); - auto canBuildLocally = amWilling + /* It would be possible to build locally after some builds clear out, + so don't show the warning now: */ + bool couldBuildLocally = maxBuildJobs > 0 && ( neededSystem == settings.thisSystem || settings.extraPlatforms.get().count(neededSystem) > 0) && allSupportedLocally(*store, requiredFeatures); + /* It's possible to build this locally right now: */ + bool canBuildLocally = amWilling && couldBuildLocally; /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); @@ -186,12 +191,12 @@ // build the hint template. std::string errorText = "Failed to find a machine for remote build!\n" - "derivation: %s\nrequired (system, features): (%s, %s)"; + "derivation: %s\nrequired (system, features): (%s, [%s])"; errorText += "\n%s available machines:"; errorText += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)"; for (unsigned int i = 0; i < machines.size(); ++i) - errorText += "\n(%s, %s, %s, %s)"; + errorText += "\n([%s], %s, [%s], [%s])"; // add the template values. std::string drvstr; @@ -214,7 +219,7 @@ % concatStringsSep(", ", m.supportedFeatures) % concatStringsSep(", ", m.mandatoryFeatures); - printMsg(canBuildLocally ? lvlChatty : lvlWarn, error); + printMsg(couldBuildLocally ? lvlChatty : lvlWarn, error.str()); std::cerr << "# decline\n"; } @@ -253,6 +258,8 @@ connected: close(5); + assert(sshStore); + std::cerr << "# accept\n" << storeUri << "\n"; auto inputs = readStrings(source); @@ -281,33 +288,61 @@ uploadLock = -1; auto drv = store->readDerivation(*drvPath); - auto outputHashes = staticOutputHashes(*store, drv); - // Hijack the inputs paths of the derivation to include all the paths - // that come from the `inputDrvs` set. - // We don’t do that for the derivations whose `inputDrvs` is empty - // because - // 1. It’s not needed - // 2. Changing the `inputSrcs` set changes the associated output ids, - // which break CA derivations - if (!drv.inputDrvs.empty()) - drv.inputSrcs = store->parseStorePathSet(inputs); + std::optional optResult; - auto result = sshStore->buildDerivation(*drvPath, drv); + // If we don't know whether we are trusted (e.g. `ssh://` + // stores), we assume we are. This is necessary for backwards + // compat. + bool trustedOrLegacy = ({ + std::optional trusted = sshStore->isTrustedClient(); + !trusted || *trusted; + }); + + // See the very large comment in `case wopBuildDerivation:` in + // `src/libstore/daemon.cc` that explains the trust model here. + // + // This condition mirrors that: that code enforces the "rules" outlined there; + // we do the best we can given those "rules". + if (trustedOrLegacy || drv.type().isCA()) { + // Hijack the inputs paths of the derivation to include all + // the paths that come from the `inputDrvs` set. We don’t do + // that for the derivations whose `inputDrvs` is empty + // because: + // + // 1. It’s not needed + // + // 2. Changing the `inputSrcs` set changes the associated + // output ids, which break CA derivations + if (!drv.inputDrvs.empty()) + drv.inputSrcs = store->parseStorePathSet(inputs); + optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); + auto & result = *optResult; + if (!result.success()) + throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); + } else { + copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); + auto res = sshStore->buildPathsWithResults({ DerivedPath::Built { *drvPath, OutputsSpec::All {} } }); + // One path to build should produce exactly one build result + assert(res.size() == 1); + optResult = std::move(res[0]); + } - if (!result.success()) - throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); + auto outputHashes = staticOutputHashes(*store, drv); std::set missingRealisations; StorePathSet missingPaths; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { for (auto & outputName : wantedOutputs) { auto thisOutputHash = outputHashes.at(outputName); auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; if (!store->queryRealisation(thisOutputId)) { debug("missing output %s", outputName); - assert(result.builtOutputs.count(thisOutputId)); - auto newRealisation = result.builtOutputs.at(thisOutputId); + assert(optResult); + auto & result = *optResult; + auto i = result.builtOutputs.find(outputName); + assert(i != result.builtOutputs.end()); + auto & newRealisation = i->second; missingRealisations.insert(newRealisation); missingPaths.insert(newRealisation.outPath); } @@ -332,7 +367,7 @@ for (auto & realisation : missingRealisations) { // Should hold, because if the feature isn't enabled the set // of missing realisations should be empty - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); store->registerDrvOutput(realisation); } diff -Nru nix-2.8.0/src/libcmd/command.cc nix-2.16.1+dfsg/src/libcmd/command.cc --- nix-2.8.0/src/libcmd/command.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/command.cc 2023-06-06 10:55:23.000000000 +0000 @@ -4,6 +4,7 @@ #include "derivations.hh" #include "nixexpr.hh" #include "profiles.hh" +#include "repl.hh" #include @@ -86,6 +87,12 @@ EvalCommand::EvalCommand() { + addFlag({ + .longName = "debugger", + .description = "Start an interactive environment if evaluation fails.", + .category = MixEvalArgs::category, + .handler = {&startReplOnEvalErrors, true}, + }); } EvalCommand::~EvalCommand() @@ -103,7 +110,7 @@ ref EvalCommand::getEvalState() { - if (!evalState) + if (!evalState) { evalState = #if HAVE_BOEHMGC std::allocate_shared(traceable_allocator(), @@ -113,9 +120,26 @@ searchPath, getEvalStore(), getStore()) #endif ; + + evalState->repair = repair; + + if (startReplOnEvalErrors) { + evalState->debugRepl = &AbstractNixRepl::runSimple; + }; + } return ref(evalState); } +MixOperateOnOptions::MixOperateOnOptions() +{ + addFlag({ + .longName = "derivation", + .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .category = installablesCategory, + .handler = {&operateOn, OperateOn::Derivation}, + }); +} + BuiltPathsCommand::BuiltPathsCommand(bool recursive) : recursive(recursive) { @@ -143,7 +167,7 @@ }); } -void BuiltPathsCommand::run(ref store) +void BuiltPathsCommand::run(ref store, Installables && installables) { BuiltPaths paths; if (all) { @@ -189,7 +213,7 @@ run(store, std::move(sorted)); } -void StorePathCommand::run(ref store, std::vector && storePaths) +void StorePathCommand::run(ref store, StorePaths && storePaths) { if (storePaths.size() != 1) throw UsageError("this command requires exactly one store path"); @@ -197,25 +221,11 @@ run(store, *storePaths.begin()); } -Strings editorFor(const Pos & pos) -{ - auto editor = getEnv("EDITOR").value_or("cat"); - auto args = tokenizeString(editor); - if (pos.line > 0 && ( - editor.find("emacs") != std::string::npos || - editor.find("nano") != std::string::npos || - editor.find("vim") != std::string::npos || - editor.find("kak") != std::string::npos)) - args.push_back(fmt("+%d", pos.line)); - args.push_back(pos.file); - return args; -} - MixProfile::MixProfile() { addFlag({ .longName = "profile", - .description = "The profile to update.", + .description = "The profile to operate on.", .labels = {"path"}, .handler = {&profile}, .completer = completePath @@ -238,7 +248,7 @@ { if (!profile) return; - std::vector result; + StorePaths result; for (auto & buildable : buildables) { std::visit(overloaded { diff -Nru nix-2.8.0/src/libcmd/command.hh nix-2.16.1+dfsg/src/libcmd/command.hh --- nix-2.8.0/src/libcmd/command.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/command.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,7 @@ #pragma once +///@file -#include "installables.hh" +#include "installable-value.hh" #include "args.hh" #include "common-eval-args.hh" #include "path.hh" @@ -18,17 +19,21 @@ struct Pos; class Store; +static constexpr Command::Category catHelp = -1; static constexpr Command::Category catSecondary = 100; static constexpr Command::Category catUtility = 101; static constexpr Command::Category catNixInstallation = 102; -static constexpr auto installablesCategory = "Options that change the interpretation of installables"; +static constexpr auto installablesCategory = "Options that change the interpretation of [installables](@docroot@/command-ref/new-cli/nix.md#installables)"; struct NixMultiCommand : virtual MultiCommand, virtual Command { nlohmann::json toJSON() override; }; +// For the overloaded run methods +#pragma GCC diagnostic ignored "-Woverloaded-virtual" + /* A command that requires a Nix store. */ struct StoreCommand : virtual Command { @@ -57,6 +62,9 @@ struct EvalCommand : virtual StoreCommand, MixEvalArgs { + bool startReplOnEvalErrors = false; + bool ignoreExceptionsDuringTry = false; + EvalCommand(); ~EvalCommand(); @@ -75,27 +83,29 @@ { flake::LockFlags lockFlags; + std::optional needsFlakeInputCompletion = {}; + MixFlakeOptions(); - virtual std::optional getFlakeRefForCompletion() + virtual std::vector getFlakesForCompletion() { return {}; } + + void completeFlakeInput(std::string_view prefix); + + void completionHook() override; }; struct SourceExprCommand : virtual Args, MixFlakeOptions { std::optional file; std::optional expr; - bool readOnlyMode = false; - - // FIXME: move this; not all commands (e.g. 'nix run') use it. - OperateOn operateOn = OperateOn::Output; - SourceExprCommand(bool supportReadOnlyMode = false); + SourceExprCommand(); - std::vector> parseInstallables( + Installables parseInstallables( ref store, std::vector ss); - std::shared_ptr parseInstallable( + ref parseInstallable( ref store, const std::string & installable); virtual Strings getDefaultFlakeAttrPaths(); @@ -105,37 +115,52 @@ void completeInstallable(std::string_view prefix); }; -/* A command that operates on a list of "installables", which can be - store paths, attribute paths, Nix expressions, etc. */ -struct InstallablesCommand : virtual Args, SourceExprCommand +struct MixReadOnlyOption : virtual Args { - std::vector> installables; + MixReadOnlyOption(); +}; - InstallablesCommand(); +/* Like InstallablesCommand but the installables are not loaded */ +struct RawInstallablesCommand : virtual Args, SourceExprCommand +{ + RawInstallablesCommand(); - void prepare() override; + virtual void run(ref store, std::vector && rawInstallables) = 0; - virtual bool useDefaultInstallables() { return true; } + void run(ref store) override; - std::optional getFlakeRefForCompletion() override; + // FIXME make const after CmdRepl's override is fixed up + virtual void applyDefaultInstallables(std::vector & rawInstallables); + + bool readFromStdIn = false; + + std::vector getFlakesForCompletion() override; private: - std::vector _installables; + std::vector rawInstallables; +}; +/* A command that operates on a list of "installables", which can be + store paths, attribute paths, Nix expressions, etc. */ +struct InstallablesCommand : RawInstallablesCommand +{ + virtual void run(ref store, Installables && installables) = 0; + + void run(ref store, std::vector && rawInstallables) override; }; /* A command that operates on exactly one "installable" */ struct InstallableCommand : virtual Args, SourceExprCommand { - std::shared_ptr installable; + InstallableCommand(); - InstallableCommand(bool supportReadOnlyMode = false); + virtual void run(ref store, ref installable) = 0; - void prepare() override; + void run(ref store) override; - std::optional getFlakeRefForCompletion() override + std::vector getFlakesForCompletion() override { - return parseFlakeRefWithFragment(_installable, absPath(".")).first; + return {_installable}; } private: @@ -143,8 +168,15 @@ std::string _installable{"."}; }; +struct MixOperateOnOptions : virtual Args +{ + OperateOn operateOn = OperateOn::Output; + + MixOperateOnOptions(); +}; + /* A command that operates on zero or more store paths. */ -struct BuiltPathsCommand : public InstallablesCommand +struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions { private: @@ -159,22 +191,18 @@ BuiltPathsCommand(bool recursive = false); - using StoreCommand::run; - virtual void run(ref store, BuiltPaths && paths) = 0; - void run(ref store) override; + void run(ref store, Installables && installables) override; - bool useDefaultInstallables() override { return !all; } + void applyDefaultInstallables(std::vector & rawInstallables) override; }; struct StorePathsCommand : public BuiltPathsCommand { StorePathsCommand(bool recursive = false); - using BuiltPathsCommand::run; - - virtual void run(ref store, std::vector && storePaths) = 0; + virtual void run(ref store, StorePaths && storePaths) = 0; void run(ref store, BuiltPaths && paths) override; }; @@ -182,11 +210,9 @@ /* A command that operates on exactly one store path. */ struct StorePathCommand : public StorePathsCommand { - using StorePathsCommand::run; - virtual void run(ref store, const StorePath & storePath) = 0; - void run(ref store, std::vector && storePaths) override; + void run(ref store, StorePaths && storePaths) override; }; /* A helper class for registering commands globally. */ @@ -217,10 +243,6 @@ return RegisterCommand(std::move(name), [](){ return make_ref(); }); } -/* Helper function to generate args that invoke $EDITOR on - filename:lineno. */ -Strings editorFor(const Pos & pos); - struct MixProfile : virtual StoreCommand { std::optional profile; diff -Nru nix-2.8.0/src/libcmd/command-installable-value.cc nix-2.16.1+dfsg/src/libcmd/command-installable-value.cc --- nix-2.8.0/src/libcmd/command-installable-value.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/command-installable-value.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,11 @@ +#include "command-installable-value.hh" + +namespace nix { + +void InstallableValueCommand::run(ref store, ref installable) +{ + auto installableValue = InstallableValue::require(installable); + run(store, installableValue); +} + +} diff -Nru nix-2.8.0/src/libcmd/command-installable-value.hh nix-2.16.1+dfsg/src/libcmd/command-installable-value.hh --- nix-2.8.0/src/libcmd/command-installable-value.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/command-installable-value.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,23 @@ +#pragma once +///@file + +#include "installable-value.hh" +#include "command.hh" + +namespace nix { + +/** + * An InstallableCommand where the single positional argument must be an + * InstallableValue in particular. + */ +struct InstallableValueCommand : InstallableCommand +{ + /** + * Entry point to this command + */ + virtual void run(ref store, ref installable) = 0; + + void run(ref store, ref installable) override; +}; + +} diff -Nru nix-2.8.0/src/libcmd/common-eval-args.cc nix-2.16.1+dfsg/src/libcmd/common-eval-args.cc --- nix-2.8.0/src/libcmd/common-eval-args.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/common-eval-args.cc 2023-06-06 10:55:23.000000000 +0000 @@ -13,8 +13,6 @@ MixEvalArgs::MixEvalArgs() { - auto category = "Common evaluation options"; - addFlag({ .longName = "arg", .description = "Pass the value *expr* as the argument *name* to Nix functions.", @@ -34,7 +32,77 @@ addFlag({ .longName = "include", .shortName = 'I', - .description = "Add *path* to the list of locations used to look up `<...>` file names.", + .description = R"( + Add *path* to the Nix search path. The Nix search path is + initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment + variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle + brackets (i.e., ``). + + For instance, passing + + ``` + -I /home/eelco/Dev + -I /etc/nixos + ``` + + will cause Nix to look for paths relative to `/home/eelco/Dev` and + `/etc/nixos`, in that order. This is equivalent to setting the + `NIX_PATH` environment variable to + + ``` + /home/eelco/Dev:/etc/nixos + ``` + + It is also possible to match paths against a prefix. For example, + passing + + ``` + -I nixpkgs=/home/eelco/Dev/nixpkgs-branch + -I /etc/nixos + ``` + + will cause Nix to search for `` in + `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. + + If a path in the Nix search path starts with `http://` or `https://`, + it is interpreted as the URL of a tarball that will be downloaded and + unpacked to a temporary location. The tarball must consist of a single + top-level directory. For example, passing + + ``` + -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz + ``` + + tells Nix to download and use the current contents of the `master` + branch in the `nixpkgs` repository. + + The URLs of the tarballs from the official `nixos.org` channels + (see [the manual page for `nix-channel`](../nix-channel.md)) can be + abbreviated as `channel:`. For instance, the + following two flags are equivalent: + + ``` + -I nixpkgs=channel:nixos-21.05 + -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz + ``` + + You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the + search path. For instance, + + ``` + -I nixpkgs=flake:nixpkgs + ``` + + specifies that the prefix `nixpkgs` shall refer to the source tree + downloaded from the `nixpkgs` entry in the flake registry. Similarly, + + ``` + -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 + ``` + + makes `` refer to a particular branch of the + `NixOS/nixpkgs` repository on GitHub. + )", .category = category, .labels = {"path"}, .handler = {[&](std::string s) { searchPath.push_back(s); }} @@ -68,7 +136,11 @@ addFlag({ .longName = "eval-store", - .description = "The Nix store to use for evaluations.", + .description = + R"( + The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them. + )", .category = category, .labels = {"store-url"}, .handler = {&evalStoreUrl}, @@ -81,7 +153,7 @@ for (auto & i : autoArgs) { auto v = state.allocValue(); if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), absPath("."))); + state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath(CanonPath::fromCwd()))); else v->mkString(((std::string_view) i.second).substr(1)); res.insert(state.symbols.create(i.first), v); @@ -89,17 +161,28 @@ return res.finish(); } -Path lookupFileArg(EvalState & state, std::string_view s) +SourcePath lookupFileArg(EvalState & state, std::string_view s) { - if (isUri(s)) { - return state.store->toRealPath( - fetchers::downloadTarball( - state.store, resolveUri(s), "source", false).first.storePath); - } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + if (EvalSettings::isPseudoUrl(s)) { + auto storePath = fetchers::downloadTarball( + state.store, EvalSettings::resolvePseudoUrl(s), "source", false).first.storePath; + return state.rootPath(CanonPath(state.store->toRealPath(storePath))); + } + + else if (hasPrefix(s, "flake:")) { + experimentalFeatureSettings.require(Xp::Flakes); + auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; + return state.rootPath(CanonPath(state.store->toRealPath(storePath))); + } + + else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p(s.substr(1, s.size() - 2)); return state.findFile(p); - } else - return absPath(std::string(s)); + } + + else + return state.rootPath(CanonPath::fromCwd(s)); } } diff -Nru nix-2.8.0/src/libcmd/common-eval-args.hh nix-2.16.1+dfsg/src/libcmd/common-eval-args.hh --- nix-2.8.0/src/libcmd/common-eval-args.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/common-eval-args.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,15 +1,20 @@ #pragma once +///@file #include "args.hh" +#include "common-args.hh" namespace nix { class Store; class EvalState; class Bindings; +struct SourcePath; -struct MixEvalArgs : virtual Args +struct MixEvalArgs : virtual Args, virtual MixRepair { + static constexpr auto category = "Common evaluation options"; + MixEvalArgs(); Bindings * getAutoArgs(EvalState & state); @@ -22,6 +27,6 @@ std::map autoArgs; }; -Path lookupFileArg(EvalState & state, std::string_view s); +SourcePath lookupFileArg(EvalState & state, std::string_view s); } diff -Nru nix-2.8.0/src/libcmd/editor-for.cc nix-2.16.1+dfsg/src/libcmd/editor-for.cc --- nix-2.8.0/src/libcmd/editor-for.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/editor-for.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,23 @@ +#include "util.hh" +#include "editor-for.hh" + +namespace nix { + +Strings editorFor(const SourcePath & file, uint32_t line) +{ + auto path = file.getPhysicalPath(); + if (!path) + throw Error("cannot open '%s' in an editor because it has no physical path", file); + auto editor = getEnv("EDITOR").value_or("cat"); + auto args = tokenizeString(editor); + if (line > 0 && ( + editor.find("emacs") != std::string::npos || + editor.find("nano") != std::string::npos || + editor.find("vim") != std::string::npos || + editor.find("kak") != std::string::npos)) + args.push_back(fmt("+%d", line)); + args.push_back(path->abs()); + return args; +} + +} diff -Nru nix-2.8.0/src/libcmd/editor-for.hh nix-2.16.1+dfsg/src/libcmd/editor-for.hh --- nix-2.8.0/src/libcmd/editor-for.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/editor-for.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,15 @@ +#pragma once +///@file + +#include "types.hh" +#include "input-accessor.hh" + +namespace nix { + +/** + * Helper function to generate args that invoke $EDITOR on + * filename:lineno. + */ +Strings editorFor(const SourcePath & file, uint32_t line); + +} diff -Nru nix-2.8.0/src/libcmd/installable-attr-path.cc nix-2.16.1+dfsg/src/libcmd/installable-attr-path.cc --- nix-2.8.0/src/libcmd/installable-attr-path.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/installable-attr-path.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,121 @@ +#include "globals.hh" +#include "installable-attr-path.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "flake/flake.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include +#include + +#include + +namespace nix { + +InstallableAttrPath::InstallableAttrPath( + ref state, + SourceExprCommand & cmd, + Value * v, + const std::string & attrPath, + ExtendedOutputsSpec extendedOutputsSpec) + : InstallableValue(state) + , cmd(cmd) + , v(allocRootValue(v)) + , attrPath(attrPath) + , extendedOutputsSpec(std::move(extendedOutputsSpec)) +{ } + +std::pair InstallableAttrPath::toValue(EvalState & state) +{ + auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); + state.forceValue(*vRes, pos); + return {vRes, pos}; +} + +DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() +{ + auto [v, pos] = toValue(*state); + + if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths( + *v, + pos, + fmt("while evaluating the attribute '%s'", attrPath))) + { + return { *derivedPathWithInfo }; + } + + Bindings & autoArgs = *cmd.getAutoArgs(*state); + + DrvInfos drvInfos; + getDerivations(*state, *v, "", autoArgs, drvInfos, false); + + // Backward compatibility hack: group results by drvPath. This + // helps keep .all output together. + std::map byDrvPath; + + for (auto & drvInfo : drvInfos) { + auto drvPath = drvInfo.queryDrvPath(); + if (!drvPath) + throw Error("'%s' is not a derivation", what()); + + auto newOutputs = std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { + std::set outputsToInstall; + for (auto & output : drvInfo.queryOutputs(false, true)) + outputsToInstall.insert(output.first); + return OutputsSpec::Names { std::move(outputsToInstall) }; + }, + [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { + return e; + }, + }, extendedOutputsSpec.raw()); + + auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs); + + if (!didInsert) + iter->second = iter->second.union_(newOutputs); + } + + DerivedPathsWithInfo res; + for (auto & [drvPath, outputs] : byDrvPath) + res.push_back({ + .path = DerivedPath::Built { + .drvPath = drvPath, + .outputs = outputs, + }, + .info = make_ref(ExtraPathInfoValue::Value { + /* FIXME: reconsider backwards compatibility above + so we can fill in this info. */ + }), + }); + + return res; +} + +InstallableAttrPath InstallableAttrPath::parse( + ref state, + SourceExprCommand & cmd, + Value * v, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec) +{ + return { + state, cmd, v, + prefix == "." ? "" : std::string { prefix }, + extendedOutputsSpec + }; +} + +} diff -Nru nix-2.8.0/src/libcmd/installable-attr-path.hh nix-2.16.1+dfsg/src/libcmd/installable-attr-path.hh --- nix-2.8.0/src/libcmd/installable-attr-path.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/installable-attr-path.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,59 @@ +#pragma once +///@file + +#include "globals.hh" +#include "installable-value.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include +#include + +#include + +namespace nix { + +class InstallableAttrPath : public InstallableValue +{ + SourceExprCommand & cmd; + RootValue v; + std::string attrPath; + ExtendedOutputsSpec extendedOutputsSpec; + + InstallableAttrPath( + ref state, + SourceExprCommand & cmd, + Value * v, + const std::string & attrPath, + ExtendedOutputsSpec extendedOutputsSpec); + + std::string what() const override { return attrPath; }; + + std::pair toValue(EvalState & state) override; + + DerivedPathsWithInfo toDerivedPaths() override; + +public: + + static InstallableAttrPath parse( + ref state, + SourceExprCommand & cmd, + Value * v, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec); +}; + +} diff -Nru nix-2.8.0/src/libcmd/installable-derived-path.cc nix-2.16.1+dfsg/src/libcmd/installable-derived-path.cc --- nix-2.8.0/src/libcmd/installable-derived-path.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/installable-derived-path.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,70 @@ +#include "installable-derived-path.hh" +#include "derivations.hh" + +namespace nix { + +std::string InstallableDerivedPath::what() const +{ + return derivedPath.to_string(*store); +} + +DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() +{ + return {{ + .path = derivedPath, + .info = make_ref(), + }}; +} + +std::optional InstallableDerivedPath::getStorePath() +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + return bfd.drvPath; + }, + [&](const DerivedPath::Opaque & bo) { + return bo.path; + }, + }, derivedPath.raw()); +} + +InstallableDerivedPath InstallableDerivedPath::parse( + ref store, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec) +{ + auto derivedPath = std::visit(overloaded { + // If the user did not use ^, we treat the output more + // liberally: we accept a symlink chain or an actual + // store path. + [&](const ExtendedOutputsSpec::Default &) -> DerivedPath { + auto storePath = store->followLinksToStorePath(prefix); + // Remove this prior to stabilizing the new CLI. + if (storePath.isDerivation()) { + auto oldDerivedPath = DerivedPath::Built { + .drvPath = storePath, + .outputs = OutputsSpec::All { }, + }; + warn( + "The interpretation of store paths arguments ending in `.drv` recently changed. If this command is now failing try again with '%s'", + oldDerivedPath.to_string(*store)); + }; + return DerivedPath::Opaque { + .path = std::move(storePath), + }; + }, + // If the user did use ^, we just do exactly what is written. + [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + return DerivedPath::Built { + .drvPath = store->parseStorePath(prefix), + .outputs = outputSpec, + }; + }, + }, extendedOutputsSpec.raw()); + return InstallableDerivedPath { + store, + std::move(derivedPath), + }; +} + +} diff -Nru nix-2.8.0/src/libcmd/installable-derived-path.hh nix-2.16.1+dfsg/src/libcmd/installable-derived-path.hh --- nix-2.8.0/src/libcmd/installable-derived-path.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/installable-derived-path.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,29 @@ +#pragma once +///@file + +#include "installables.hh" + +namespace nix { + +struct InstallableDerivedPath : Installable +{ + ref store; + DerivedPath derivedPath; + + InstallableDerivedPath(ref store, DerivedPath && derivedPath) + : store(store), derivedPath(std::move(derivedPath)) + { } + + std::string what() const override; + + DerivedPathsWithInfo toDerivedPaths() override; + + std::optional getStorePath() override; + + static InstallableDerivedPath parse( + ref store, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec); +}; + +} diff -Nru nix-2.8.0/src/libcmd/installable-flake.cc nix-2.16.1+dfsg/src/libcmd/installable-flake.cc --- nix-2.8.0/src/libcmd/installable-flake.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/installable-flake.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,222 @@ +#include "globals.hh" +#include "installable-flake.hh" +#include "installable-derived-path.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "flake/flake.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include +#include + +#include + +namespace nix { + +std::vector InstallableFlake::getActualAttrPaths() +{ + std::vector res; + + for (auto & prefix : prefixes) + res.push_back(prefix + *attrPaths.begin()); + + for (auto & s : attrPaths) + res.push_back(s); + + return res; +} + +Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) +{ + auto vFlake = state.allocValue(); + + callFlake(state, lockedFlake, *vFlake); + + auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); }); + + return aOutputs->value; +} + +static std::string showAttrPaths(const std::vector & paths) +{ + std::string s; + for (const auto & [n, i] : enumerate(paths)) { + if (n > 0) s += n + 1 == paths.size() ? " or " : ", "; + s += '\''; s += i; s += '\''; + } + return s; +} + +InstallableFlake::InstallableFlake( + SourceExprCommand * cmd, + ref state, + FlakeRef && flakeRef, + std::string_view fragment, + ExtendedOutputsSpec extendedOutputsSpec, + Strings attrPaths, + Strings prefixes, + const flake::LockFlags & lockFlags) + : InstallableValue(state), + flakeRef(flakeRef), + attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), + prefixes(fragment == "" ? Strings{} : prefixes), + extendedOutputsSpec(std::move(extendedOutputsSpec)), + lockFlags(lockFlags) +{ + if (cmd && cmd->getAutoArgs(*state)->size()) + throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); +} + +DerivedPathsWithInfo InstallableFlake::toDerivedPaths() +{ + Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); + + auto attr = getCursor(*state); + + auto attrPath = attr->getAttrPathStr(); + + if (!attr->isDerivation()) { + + // FIXME: use eval cache? + auto v = attr->forceValue(); + + if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths( + v, + noPos, + fmt("while evaluating the flake output attribute '%s'", attrPath))) + { + return { *derivedPathWithInfo }; + } + else + throw Error("flake output attribute '%s' is not a derivation or path", attrPath); + } + + auto drvPath = attr->forceDerivation(); + + std::optional priority; + + if (attr->maybeGetAttr(state->sOutputSpecified)) { + } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + if (auto aPriority = aMeta->maybeGetAttr("priority")) + priority = aPriority->getInt(); + } + + return {{ + .path = DerivedPath::Built { + .drvPath = std::move(drvPath), + .outputs = std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { + std::set outputsToInstall; + if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { + if (aOutputSpecified->getBool()) { + if (auto aOutputName = attr->maybeGetAttr("outputName")) + outputsToInstall = { aOutputName->getString() }; + } + } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) + for (auto & s : aOutputsToInstall->getListOfStrings()) + outputsToInstall.insert(s); + } + + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); + + return OutputsSpec::Names { std::move(outputsToInstall) }; + }, + [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { + return e; + }, + }, extendedOutputsSpec.raw()), + }, + .info = make_ref( + ExtraPathInfoValue::Value { + .priority = priority, + .attrPath = attrPath, + .extendedOutputsSpec = extendedOutputsSpec, + }, + ExtraPathInfoFlake::Flake { + .originalRef = flakeRef, + .resolvedRef = getLockedFlake()->flake.lockedRef, + }), + }}; +} + +std::pair InstallableFlake::toValue(EvalState & state) +{ + return {&getCursor(state)->forceValue(), noPos}; +} + +std::vector> +InstallableFlake::getCursors(EvalState & state) +{ + auto evalCache = openEvalCache(state, getLockedFlake()); + + auto root = evalCache->getRoot(); + + std::vector> res; + + Suggestions suggestions; + auto attrPaths = getActualAttrPaths(); + + for (auto & attrPath : attrPaths) { + debug("trying flake output attribute '%s'", attrPath); + + auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); + if (attr) { + res.push_back(ref(*attr)); + } else { + suggestions += attr.getSuggestions(); + } + } + + if (res.size() == 0) + throw Error( + suggestions, + "flake '%s' does not provide attribute %s", + flakeRef, + showAttrPaths(attrPaths)); + + return res; +} + +std::shared_ptr InstallableFlake::getLockedFlake() const +{ + if (!_lockedFlake) { + flake::LockFlags lockFlagsApplyConfig = lockFlags; + // FIXME why this side effect? + lockFlagsApplyConfig.applyNixConfig = true; + _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + } + return _lockedFlake; +} + +FlakeRef InstallableFlake::nixpkgsFlakeRef() const +{ + auto lockedFlake = getLockedFlake(); + + if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { + if (auto lockedNode = std::dynamic_pointer_cast(nixpkgsInput)) { + debug("using nixpkgs flake '%s'", lockedNode->lockedRef); + return std::move(lockedNode->lockedRef); + } + } + + return defaultNixpkgsFlakeRef(); +} + +} diff -Nru nix-2.8.0/src/libcmd/installable-flake.hh nix-2.16.1+dfsg/src/libcmd/installable-flake.hh --- nix-2.8.0/src/libcmd/installable-flake.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/installable-flake.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,90 @@ +#pragma once +///@file + +#include "installable-value.hh" + +namespace nix { + +/** + * Extra info about a \ref DerivedPath "derived path" that ultimately + * come from a Flake. + * + * Invariant: every ExtraPathInfo gotten from an InstallableFlake should + * be possible to downcast to an ExtraPathInfoFlake. + */ +struct ExtraPathInfoFlake : ExtraPathInfoValue +{ + /** + * Extra struct to get around C++ designated initializer limitations + */ + struct Flake { + FlakeRef originalRef; + FlakeRef resolvedRef; + }; + + Flake flake; + + ExtraPathInfoFlake(Value && v, Flake && f) + : ExtraPathInfoValue(std::move(v)), flake(f) + { } +}; + +struct InstallableFlake : InstallableValue +{ + FlakeRef flakeRef; + Strings attrPaths; + Strings prefixes; + ExtendedOutputsSpec extendedOutputsSpec; + const flake::LockFlags & lockFlags; + mutable std::shared_ptr _lockedFlake; + + InstallableFlake( + SourceExprCommand * cmd, + ref state, + FlakeRef && flakeRef, + std::string_view fragment, + ExtendedOutputsSpec extendedOutputsSpec, + Strings attrPaths, + Strings prefixes, + const flake::LockFlags & lockFlags); + + std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } + + std::vector getActualAttrPaths(); + + Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); + + DerivedPathsWithInfo toDerivedPaths() override; + + std::pair toValue(EvalState & state) override; + + /** + * Get a cursor to every attrpath in getActualAttrPaths() that + * exists. However if none exists, throw an exception. + */ + std::vector> + getCursors(EvalState & state) override; + + std::shared_ptr getLockedFlake() const; + + FlakeRef nixpkgsFlakeRef() const; +}; + +/** + * Default flake ref for referring to Nixpkgs. For flakes that don't + * have their own Nixpkgs input, or other installables. + * + * It is a layer violation for Nix to know about Nixpkgs; currently just + * `nix develop` does. Be wary of using this / + * `InstallableFlake::nixpkgsFlakeRef` more places. + */ +static inline FlakeRef defaultNixpkgsFlakeRef() +{ + return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); +} + +ref openEvalCache( + EvalState & state, + std::shared_ptr lockedFlake); + +} diff -Nru nix-2.8.0/src/libcmd/installables.cc nix-2.16.1+dfsg/src/libcmd/installables.cc --- nix-2.8.0/src/libcmd/installables.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/installables.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,5 +1,9 @@ #include "globals.hh" #include "installables.hh" +#include "installable-derived-path.hh" +#include "installable-attr-path.hh" +#include "installable-flake.hh" +#include "outputs-spec.hh" #include "util.hh" #include "command.hh" #include "attr-path.hh" @@ -23,17 +27,6 @@ namespace nix { -void completeFlakeInputPath( - ref evalState, - const FlakeRef & flakeRef, - std::string_view prefix) -{ - auto flake = flake::getFlake(*evalState, flakeRef, true); - for (auto & input : flake.inputs) - if (hasPrefix(input.first, prefix)) - completions->add(input.first); -} - MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; @@ -86,8 +79,7 @@ lockFlags.inputUpdates.insert(flake::parseInputPath(s)); }}, .completer = {[&](size_t, std::string_view prefix) { - if (auto flakeRef = getFlakeRefForCompletion()) - completeFlakeInputPath(getEvalState(), *flakeRef, prefix); + needsFlakeInputCompletion = {std::string(prefix)}; }} }); @@ -103,16 +95,36 @@ parseFlakeRef(flakeRef, absPath("."), true)); }}, .completer = {[&](size_t n, std::string_view prefix) { - if (n == 0) { - if (auto flakeRef = getFlakeRefForCompletion()) - completeFlakeInputPath(getEvalState(), *flakeRef, prefix); - } else if (n == 1) { + if (n == 0) + needsFlakeInputCompletion = {std::string(prefix)}; + else if (n == 1) completeFlakeRef(getEvalState()->store, prefix); - } }} }); addFlag({ + .longName = "reference-lock-file", + .description = "Read the given lock file instead of `flake.lock` within the top-level flake.", + .category = category, + .labels = {"flake-lock-path"}, + .handler = {[&](std::string lockFilePath) { + lockFlags.referenceLockFilePath = lockFilePath; + }}, + .completer = completePath + }); + + addFlag({ + .longName = "output-lock-file", + .description = "Write the given lock file instead of `flake.lock` within the top-level flake.", + .category = category, + .labels = {"flake-lock-path"}, + .handler = {[&](std::string lockFilePath) { + lockFlags.outputLockFilePath = lockFilePath; + }}, + .completer = completePath + }); + + addFlag({ .longName = "inputs-from", .description = "Use the inputs of the specified flake as registry entries.", .category = category, @@ -139,14 +151,33 @@ }); } -SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) +void MixFlakeOptions::completeFlakeInput(std::string_view prefix) +{ + auto evalState = getEvalState(); + for (auto & flakeRefS : getFlakesForCompletion()) { + auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first; + auto flake = flake::getFlake(*evalState, flakeRef, true); + for (auto & input : flake.inputs) + if (hasPrefix(input.first, prefix)) + completions->add(input.first); + } +} + +void MixFlakeOptions::completionHook() +{ + if (auto & prefix = needsFlakeInputCompletion) + completeFlakeInput(*prefix); +} + +SourceExprCommand::SourceExprCommand() { addFlag({ .longName = "file", .shortName = 'f', .description = - "Interpret installables as attribute paths relative to the Nix expression stored in *file*. " - "If *file* is the character -, then a Nix expression will be read from standard input.", + "Interpret [*installables*](@docroot@/command-ref/new-cli/nix.md#installables) as attribute paths relative to the Nix expression stored in *file*. " + "If *file* is the character -, then a Nix expression will be read from standard input. " + "Implies `--impure`.", .category = installablesCategory, .labels = {"file"}, .handler = {&file}, @@ -155,29 +186,23 @@ addFlag({ .longName = "expr", - .description = "Interpret installables as attribute paths relative to the Nix expression *expr*.", + .description = "Interpret [*installables*](@docroot@/command-ref/new-cli/nix.md#installables) as attribute paths relative to the Nix expression *expr*.", .category = installablesCategory, .labels = {"expr"}, .handler = {&expr} }); +} +MixReadOnlyOption::MixReadOnlyOption() +{ addFlag({ - .longName = "derivation", - .description = "Operate on the store derivation rather than its outputs.", - .category = installablesCategory, - .handler = {&operateOn, OperateOn::Derivation}, + .longName = "read-only", + .description = + "Do not instantiate each evaluated derivation. " + "This improves performance, but can cause errors when accessing " + "store paths of derivations during evaluation.", + .handler = {&settings.readOnlyMode, true}, }); - - if (supportReadOnlyMode) { - addFlag({ - .longName = "read-only", - .description = - "Do not instantiate each evaluated derivation. " - "This improves performance, but can cause errors when accessing " - "store paths of derivations during evaluation.", - .handler = {&readOnlyMode, true}, - }); - } } Strings SourceExprCommand::getDefaultFlakeAttrPaths() @@ -202,55 +227,59 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) { - if (file) { - completionType = ctAttrs; + try { + if (file) { + completionType = ctAttrs; - evalSettings.pureEval = false; - auto state = getEvalState(); - Expr *e = state->parseExprFromFile( - resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file))) - ); - - Value root; - state->eval(e, root); - - auto autoArgs = getAutoArgs(*state); - - std::string prefix_ = std::string(prefix); - auto sep = prefix_.rfind('.'); - std::string searchWord; - if (sep != std::string::npos) { - searchWord = prefix_.substr(sep + 1, std::string::npos); - prefix_ = prefix_.substr(0, sep); - } else { - searchWord = prefix_; - prefix_ = ""; - } + evalSettings.pureEval = false; + auto state = getEvalState(); + Expr *e = state->parseExprFromFile( + resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file))) + ); + + Value root; + state->eval(e, root); + + auto autoArgs = getAutoArgs(*state); + + std::string prefix_ = std::string(prefix); + auto sep = prefix_.rfind('.'); + std::string searchWord; + if (sep != std::string::npos) { + searchWord = prefix_.substr(sep + 1, std::string::npos); + prefix_ = prefix_.substr(0, sep); + } else { + searchWord = prefix_; + prefix_ = ""; + } - auto [v, pos] = findAlongAttrPath(*state, prefix_, *autoArgs, root); - Value &v1(*v); - state->forceValue(v1, pos); - Value v2; - state->autoCallFunction(*autoArgs, v1, v2); - - if (v2.type() == nAttrs) { - for (auto & i : *v2.attrs) { - std::string name = i.name; - if (name.find(searchWord) == 0) { - if (prefix_ == "") - completions->add(name); - else - completions->add(prefix_ + "." + name); + auto [v, pos] = findAlongAttrPath(*state, prefix_, *autoArgs, root); + Value &v1(*v); + state->forceValue(v1, pos); + Value v2; + state->autoCallFunction(*autoArgs, v1, v2); + + if (v2.type() == nAttrs) { + for (auto & i : *v2.attrs) { + std::string name = state->symbols[i.name]; + if (name.find(searchWord) == 0) { + if (prefix_ == "") + completions->add(name); + else + completions->add(prefix_ + "." + name); + } } } + } else { + completeFlakeRefWithFragment( + getEvalState(), + lockFlags, + getDefaultFlakeAttrPathPrefixes(), + getDefaultFlakeAttrPaths(), + prefix); } - } else { - completeFlakeRefWithFragment( - getEvalState(), - lockFlags, - getDefaultFlakeAttrPathPrefixes(), - getDefaultFlakeAttrPaths(), - prefix); + } catch (EvalError&) { + // Don't want eval errors to mess-up with the completion engine, so let's just swallow them } } @@ -291,7 +320,7 @@ std::string lastAttr; if (!attrPath.empty() && !hasSuffix(attrPathS, ".")) { - lastAttr = attrPath.back(); + lastAttr = evalState->symbols[attrPath.back()]; attrPath.pop_back(); } @@ -299,11 +328,11 @@ if (!attr) continue; for (auto & attr2 : (*attr)->getAttrs()) { - if (hasPrefix(attr2, lastAttr)) { + if (hasPrefix(evalState->symbols[attr2], lastAttr)) { auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2)); + completions->add(flakeRefS + "#" + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } } @@ -325,7 +354,7 @@ void completeFlakeRef(ref store, std::string_view prefix) { - if (!settings.isExperimentalFeatureEnabled(Xp::Flakes)) + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) return; if (prefix == "") @@ -349,7 +378,7 @@ } } -DerivedPath Installable::toDerivedPath() +DerivedPathWithInfo Installable::toDerivedPath() { auto buildables = toDerivedPaths(); if (buildables.size() != 1) @@ -357,24 +386,6 @@ return std::move(buildables[0]); } -std::vector> -Installable::getCursors(EvalState & state) -{ - auto evalCache = - std::make_shared(std::nullopt, state, - [&]() { return toValue(state).first; }); - return {evalCache->getRoot()}; -} - -ref -Installable::getCursor(EvalState & state) -{ - auto cursors = getCursors(state); - if (cursors.empty()) - throw Error("cannot find flake attribute '%s'", what()); - return cursors[0]; -} - static StorePath getDeriver( ref store, const Installable & i, @@ -387,149 +398,6 @@ return *derivers.begin(); } -struct InstallableStorePath : Installable -{ - ref store; - StorePath storePath; - - InstallableStorePath(ref store, StorePath && storePath) - : store(store), storePath(std::move(storePath)) { } - - std::string what() const override { return store->printStorePath(storePath); } - - DerivedPaths toDerivedPaths() override - { - if (storePath.isDerivation()) { - auto drv = store->readDerivation(storePath); - return { - DerivedPath::Built { - .drvPath = storePath, - .outputs = drv.outputNames(), - } - }; - } else { - return { - DerivedPath::Opaque { - .path = storePath, - } - }; - } - } - - StorePathSet toDrvPaths(ref store) override - { - if (storePath.isDerivation()) { - return {storePath}; - } else { - return {getDeriver(store, *this, storePath)}; - } - } - - std::optional getStorePath() override - { - return storePath; - } -}; - -DerivedPaths InstallableValue::toDerivedPaths() -{ - DerivedPaths res; - - std::map> drvsToOutputs; - RealisedPath::Set drvsToCopy; - - // Group by derivation, helps with .all in particular - for (auto & drv : toDerivations()) { - auto outputName = drv.outputName; - if (outputName == "") - throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath)); - drvsToOutputs[drv.drvPath].insert(outputName); - drvsToCopy.insert(drv.drvPath); - } - - for (auto & i : drvsToOutputs) - res.push_back(DerivedPath::Built { i.first, i.second }); - - return res; -} - -StorePathSet InstallableValue::toDrvPaths(ref store) -{ - StorePathSet res; - for (auto & drv : toDerivations()) - res.insert(drv.drvPath); - return res; -} - -struct InstallableAttrPath : InstallableValue -{ - SourceExprCommand & cmd; - RootValue v; - std::string attrPath; - - InstallableAttrPath(ref state, SourceExprCommand & cmd, Value * v, const std::string & attrPath) - : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath) - { } - - std::string what() const override { return attrPath; } - - std::pair toValue(EvalState & state) override - { - auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); - state.forceValue(*vRes, pos); - return {vRes, pos}; - } - - virtual std::vector toDerivations() override; -}; - -std::vector InstallableAttrPath::toDerivations() -{ - auto v = toValue(*state).first; - - Bindings & autoArgs = *cmd.getAutoArgs(*state); - - DrvInfos drvInfos; - getDerivations(*state, *v, "", autoArgs, drvInfos, false); - - std::vector res; - for (auto & drvInfo : drvInfos) { - auto drvPath = drvInfo.queryDrvPath(); - if (!drvPath) - throw Error("'%s' is not a derivation", what()); - res.push_back({ *drvPath, drvInfo.queryOutputName() }); - } - - return res; -} - -std::vector InstallableFlake::getActualAttrPaths() -{ - std::vector res; - - for (auto & prefix : prefixes) - res.push_back(prefix + *attrPaths.begin()); - - for (auto & s : attrPaths) - res.push_back(s); - - return res; -} - -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); }); - - return aOutputs->value; -} - ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake) @@ -550,7 +418,7 @@ auto vFlake = state.allocValue(); flake::callFlake(state, *lockedFlake, *vFlake); - state.forceAttrs(*vFlake, noPos); + state.forceAttrs(*vFlake, noPos, "while parsing cached flake data"); auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); assert(aOutputs); @@ -559,149 +427,10 @@ }); } -static std::string showAttrPaths(const std::vector & paths) -{ - std::string s; - for (const auto & [n, i] : enumerate(paths)) { - if (n > 0) s += n + 1 == paths.size() ? " or " : ", "; - s += '\''; s += i; s += '\''; - } - return s; -} - -InstallableFlake::InstallableFlake( - SourceExprCommand * cmd, - ref state, - FlakeRef && flakeRef, - std::string_view fragment, - Strings attrPaths, - Strings prefixes, - const flake::LockFlags & lockFlags) - : InstallableValue(state), - flakeRef(flakeRef), - attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), - prefixes(fragment == "" ? Strings{} : prefixes), - lockFlags(lockFlags) -{ - if (cmd && cmd->getAutoArgs(*state)->size()) - throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); -} - -std::tuple InstallableFlake::toDerivation() -{ - auto attr = getCursor(*state); - - auto attrPath = attr->getAttrPathStr(); - - if (!attr->isDerivation()) - throw Error("flake output attribute '%s' is not a derivation", attrPath); - - auto drvPath = attr->forceDerivation(); - - auto drvInfo = DerivationInfo { - std::move(drvPath), - attr->getAttr(state->sOutputName)->getString() - }; - - return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; -} - -std::vector InstallableFlake::toDerivations() -{ - std::vector res; - res.push_back(std::get<2>(toDerivation())); - return res; -} - -std::pair InstallableFlake::toValue(EvalState & state) -{ - return {&getCursor(state)->forceValue(), noPos}; -} - -std::vector> -InstallableFlake::getCursors(EvalState & state) -{ - auto evalCache = openEvalCache(state, - std::make_shared(lockFlake(state, flakeRef, lockFlags))); - - auto root = evalCache->getRoot(); - - std::vector> res; - - for (auto & attrPath : getActualAttrPaths()) { - auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); - if (attr) res.push_back(ref(*attr)); - } - - return res; -} - -ref InstallableFlake::getCursor(EvalState & state) -{ - auto lockedFlake = getLockedFlake(); - - auto cache = openEvalCache(state, lockedFlake); - auto root = cache->getRoot(); - - Suggestions suggestions; - - auto attrPaths = getActualAttrPaths(); - - for (auto & attrPath : attrPaths) { - debug("trying flake output attribute '%s'", attrPath); - - auto attrOrSuggestions = root->findAlongAttrPath( - parseAttrPath(state, attrPath), - true - ); - - if (!attrOrSuggestions) { - suggestions += attrOrSuggestions.getSuggestions(); - continue; - } - - return *attrOrSuggestions; - } - - throw Error( - suggestions, - "flake '%s' does not provide attribute %s", - flakeRef, - showAttrPaths(attrPaths)); -} - -std::shared_ptr InstallableFlake::getLockedFlake() const -{ - if (!_lockedFlake) { - flake::LockFlags lockFlagsApplyConfig = lockFlags; - lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); - } - return _lockedFlake; -} - -FlakeRef InstallableFlake::nixpkgsFlakeRef() const -{ - auto lockedFlake = getLockedFlake(); - - if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { - if (auto lockedNode = std::dynamic_pointer_cast(nixpkgsInput)) { - debug("using nixpkgs flake '%s'", lockedNode->lockedRef); - return std::move(lockedNode->lockedRef); - } - } - - return Installable::nixpkgsFlakeRef(); -} - -std::vector> SourceExprCommand::parseInstallables( +Installables SourceExprCommand::parseInstallables( ref store, std::vector ss) { - std::vector> result; - - if (readOnlyMode) { - settings.readOnlyMode = true; - } + Installables result; if (file || expr) { if (file && expr) @@ -716,24 +445,36 @@ if (file == "-") { auto e = state->parseStdin(); state->eval(e, *vFile); - } else if (file) + } + else if (file) state->evalFile(lookupFileArg(*state, *file), *vFile); else { - auto e = state->parseExprFromString(*expr, absPath(".")); + auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd())); state->eval(e, *vFile); } - for (auto & s : ss) - result.push_back(std::make_shared(state, *this, vFile, s == "." ? "" : s)); + for (auto & s : ss) { + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(s); + result.push_back( + make_ref( + InstallableAttrPath::parse( + state, *this, vFile, prefix, extendedOutputsSpec))); + } } else { for (auto & s : ss) { std::exception_ptr ex; - if (s.find('/') != std::string::npos) { + auto [prefix_, extendedOutputsSpec_] = ExtendedOutputsSpec::parse(s); + // To avoid clang's pedantry + auto prefix = std::move(prefix_); + auto extendedOutputsSpec = std::move(extendedOutputsSpec_); + + if (prefix.find('/') != std::string::npos) { try { - result.push_back(std::make_shared(store, store->followLinksToStorePath(s))); + result.push_back(make_ref( + InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec))); continue; } catch (BadStorePath &) { } catch (...) { @@ -743,12 +484,13 @@ } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath(".")); - result.push_back(std::make_shared( + auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(".")); + result.push_back(make_ref( this, getEvalState(), std::move(flakeRef), fragment, + extendedOutputsSpec, getDefaultFlakeAttrPaths(), getDefaultFlakeAttrPathPrefixes(), lockFlags)); @@ -764,7 +506,7 @@ return result; } -std::shared_ptr SourceExprCommand::parseInstallable( +ref SourceExprCommand::parseInstallable( ref store, const std::string & installable) { auto installables = parseInstallables(store, {installable}); @@ -772,40 +514,46 @@ return installables.front(); } -BuiltPaths Installable::build( +std::vector Installable::build( ref evalStore, ref store, Realise mode, - const std::vector> & installables, + const Installables & installables, BuildMode bMode) { - BuiltPaths res; - for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode)) - res.push_back(builtPath); + std::vector res; + for (auto & [_, builtPathWithResult] : build2(evalStore, store, mode, installables, bMode)) + res.push_back(builtPathWithResult); return res; } -std::vector, BuiltPath>> Installable::build2( +std::vector, BuiltPathWithResult>> Installable::build2( ref evalStore, ref store, Realise mode, - const std::vector> & installables, + const Installables & installables, BuildMode bMode) { if (mode == Realise::Nothing) settings.readOnlyMode = true; + struct Aux + { + ref info; + ref installable; + }; + std::vector pathsToBuild; - std::map>> backmap; + std::map> backmap; for (auto & i : installables) { for (auto b : i->toDerivedPaths()) { - pathsToBuild.push_back(b); - backmap[b].push_back(i); + pathsToBuild.push_back(b.path); + backmap[b.path].push_back({.info = b.info, .installable = i}); } } - std::vector, BuiltPath>> res; + std::vector, BuiltPathWithResult>> res; switch (mode) { @@ -814,40 +562,18 @@ printMissing(store, pathsToBuild, lvlError); for (auto & path : pathsToBuild) { - for (auto & installable : backmap[path]) { + for (auto & aux : backmap[path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { - OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive - auto drvOutputs = drv.outputsAndOptPaths(*store); - for (auto & output : bfd.outputs) { - if (!outputHashes.count(output)) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store->printStorePath(bfd.drvPath), output); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { - DrvOutput outputId { outputHashes.at(output), output }; - auto realisation = store->queryRealisation(outputId); - if (!realisation) - throw Error( - "cannot operate on an output of unbuilt " - "content-addressed derivation '%s'", - outputId.to_string()); - outputs.insert_or_assign(output, realisation->outPath); - } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - assert(drvOutputs.count(output)); - assert(drvOutputs.at(output).second); - outputs.insert_or_assign( - output, *drvOutputs.at(output).second); - } - } - res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }}); + auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); + res.push_back({aux.installable, { + .path = BuiltPath::Built { bfd.drvPath, outputs }, + .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, BuiltPath::Opaque { bo.path }}); + res.push_back({aux.installable, { + .path = BuiltPath::Opaque { bo.path }, + .info = aux.info}}); }, }, path.raw()); } @@ -856,20 +582,29 @@ break; case Realise::Outputs: { + if (settings.printMissing) + printMissing(store, pathsToBuild, lvlInfo); + for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { if (!buildResult.success()) buildResult.rethrow(); - for (auto & installable : backmap[buildResult.path]) { + for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { std::map outputs; - for (auto & path : buildResult.builtOutputs) - outputs.emplace(path.first.outputName, path.second.outPath); - res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }}); + for (auto & [outputName, realisation] : buildResult.builtOutputs) + outputs.emplace(outputName, realisation.outPath); + res.push_back({aux.installable, { + .path = BuiltPath::Built { bfd.drvPath, outputs }, + .info = aux.info, + .result = buildResult}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, BuiltPath::Opaque { bo.path }}); + res.push_back({aux.installable, { + .path = BuiltPath::Opaque { bo.path }, + .info = aux.info, + .result = buildResult}}); }, }, buildResult.path.raw()); } @@ -890,11 +625,14 @@ ref store, Realise mode, OperateOn operateOn, - const std::vector> & installables) + const Installables & installables) { - if (operateOn == OperateOn::Output) - return Installable::build(evalStore, store, mode, installables); - else { + if (operateOn == OperateOn::Output) { + BuiltPaths res; + for (auto & p : Installable::build(evalStore, store, mode, installables)) + res.push_back(p.path); + return res; + } else { if (mode == Realise::Nothing) settings.readOnlyMode = true; @@ -909,7 +647,7 @@ ref evalStore, ref store, Realise mode, OperateOn operateOn, - const std::vector> & installables) + const Installables & installables) { StorePathSet outPaths; for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) { @@ -923,7 +661,7 @@ ref evalStore, ref store, Realise mode, OperateOn operateOn, - std::shared_ptr installable) + ref installable) { auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable}); @@ -935,7 +673,7 @@ StorePathSet Installable::toDerivations( ref store, - const std::vector> & installables, + const Installables & installables, bool useDeriver) { StorePathSet drvPaths; @@ -944,50 +682,74 @@ for (const auto & b : i->toDerivedPaths()) std::visit(overloaded { [&](const DerivedPath::Opaque & bo) { - if (!useDeriver) - throw Error("argument '%s' did not evaluate to a derivation", i->what()); - drvPaths.insert(getDeriver(store, *i, bo.path)); + drvPaths.insert( + bo.path.isDerivation() + ? bo.path + : useDeriver + ? getDeriver(store, *i, bo.path) + : throw Error("argument '%s' did not evaluate to a derivation", i->what())); }, [&](const DerivedPath::Built & bfd) { drvPaths.insert(bfd.drvPath); }, - }, b.raw()); + }, b.path.raw()); return drvPaths; } -InstallablesCommand::InstallablesCommand() +RawInstallablesCommand::RawInstallablesCommand() { + addFlag({ + .longName = "stdin", + .description = "Read installables from the standard input.", + .handler = {&readFromStdIn, true} + }); + expectArgs({ .label = "installables", - .handler = {&_installables}, + .handler = {&rawInstallables}, .completer = {[&](size_t, std::string_view prefix) { completeInstallable(prefix); }} }); } -void InstallablesCommand::prepare() +void RawInstallablesCommand::applyDefaultInstallables(std::vector & rawInstallables) { - if (_installables.empty() && useDefaultInstallables()) + if (rawInstallables.empty()) { // FIXME: commands like "nix profile install" should not have a // default, probably. - _installables.push_back("."); - installables = parseInstallables(getStore(), _installables); + rawInstallables.push_back("."); + } } -std::optional InstallablesCommand::getFlakeRefForCompletion() +void RawInstallablesCommand::run(ref store) { - if (_installables.empty()) { - if (useDefaultInstallables()) - return parseFlakeRefWithFragment(".", absPath(".")).first; - return {}; + if (readFromStdIn && !isatty(STDIN_FILENO)) { + std::string word; + while (std::cin >> word) { + rawInstallables.emplace_back(std::move(word)); + } } - return parseFlakeRefWithFragment(_installables.front(), absPath(".")).first; + + applyDefaultInstallables(rawInstallables); + run(store, std::move(rawInstallables)); +} + +std::vector RawInstallablesCommand::getFlakesForCompletion() +{ + applyDefaultInstallables(rawInstallables); + return rawInstallables; } -InstallableCommand::InstallableCommand(bool supportReadOnlyMode) - : SourceExprCommand(supportReadOnlyMode) +void InstallablesCommand::run(ref store, std::vector && rawInstallables) +{ + auto installables = parseInstallables(store, rawInstallables); + run(store, std::move(installables)); +} + +InstallableCommand::InstallableCommand() + : SourceExprCommand() { expectArgs({ .label = "installable", @@ -999,9 +761,16 @@ }); } -void InstallableCommand::prepare() +void InstallableCommand::run(ref store) +{ + auto installable = parseInstallable(store, _installable); + run(store, std::move(installable)); +} + +void BuiltPathsCommand::applyDefaultInstallables(std::vector & rawInstallables) { - installable = parseInstallable(getStore(), _installable); + if (rawInstallables.empty() && !all) + rawInstallables.push_back("."); } } diff -Nru nix-2.8.0/src/libcmd/installables.hh nix-2.16.1+dfsg/src/libcmd/installables.hh --- nix-2.8.0/src/libcmd/installables.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/installables.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,108 +1,168 @@ #pragma once +///@file #include "util.hh" #include "path.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "derived-path.hh" -#include "eval.hh" #include "store-api.hh" -#include "flake/flake.hh" +#include "build-result.hh" #include namespace nix { struct DrvInfo; -struct SourceExprCommand; - -namespace eval_cache { class EvalCache; class AttrCursor; } - -struct App -{ - std::vector context; - Path program; - // FIXME: add args, sandbox settings, metadata, ... -}; - -struct UnresolvedApp -{ - App unresolved; - App resolve(ref evalStore, ref store); -}; enum class Realise { - /* Build the derivation. Postcondition: the - derivation outputs exist. */ + /** + * Build the derivation. + * + * Postcondition: the derivation outputs exist. + */ Outputs, - /* Don't build the derivation. Postcondition: the store derivation - exists. */ + /** + * Don't build the derivation. + * + * Postcondition: the store derivation exists. + */ Derivation, - /* Evaluate in dry-run mode. Postcondition: nothing. */ - // FIXME: currently unused, but could be revived if we can - // evaluate derivations in-memory. + /** + * Evaluate in dry-run mode. + * + * Postcondition: nothing. + * + * \todo currently unused, but could be revived if we can evaluate + * derivations in-memory. + */ Nothing }; -/* How to handle derivations in commands that operate on store paths. */ +/** + * How to handle derivations in commands that operate on store paths. + */ enum class OperateOn { - /* Operate on the output path. */ + /** + * Operate on the output path. + */ Output, - /* Operate on the .drv path. */ + /** + * Operate on the .drv path. + */ Derivation }; +/** + * Extra info about a DerivedPath + * + * Yes, this is empty, but that is intended. It will be sub-classed by + * the subclasses of Installable to allow those to provide more info. + * Certain commands will make use of this info. + */ +struct ExtraPathInfo +{ + virtual ~ExtraPathInfo() = default; +}; + +/** + * A DerivedPath with \ref ExtraPathInfo "any additional info" that + * commands might need from the derivation. + */ +struct DerivedPathWithInfo +{ + DerivedPath path; + ref info; +}; + +/** + * Like DerivedPathWithInfo but extending BuiltPath with \ref + * ExtraPathInfo "extra info" and also possibly the \ref BuildResult + * "result of building". + */ +struct BuiltPathWithResult +{ + BuiltPath path; + ref info; + std::optional result; +}; + +/** + * Shorthand, for less typing and helping us keep the choice of + * collection in sync. + */ +typedef std::vector DerivedPathsWithInfo; + +struct Installable; + +/** + * Shorthand, for less typing and helping us keep the choice of + * collection in sync. + */ +typedef std::vector> Installables; + +/** + * Installables are the main positional arguments for the Nix + * Command-line. + * + * This base class is very flexible, and just assumes and the + * Installable refers to a collection of \ref DerivedPath "derived paths" with + * \ref ExtraPathInfo "extra info". + */ struct Installable { virtual ~Installable() { } + /** + * What Installable is this? + * + * Prints back valid CLI syntax that would result in this same + * installable. It doesn't need to be exactly what the user wrote, + * just something that means the same thing. + */ virtual std::string what() const = 0; - virtual DerivedPaths toDerivedPaths() = 0; - - virtual StorePathSet toDrvPaths(ref store) - { - throw Error("'%s' cannot be converted to a derivation path", what()); - } - - DerivedPath toDerivedPath(); - - UnresolvedApp toApp(EvalState & state); - - virtual std::pair toValue(EvalState & state) - { - throw Error("argument '%s' cannot be evaluated", what()); - } - - /* Return a value only if this installable is a store path or a - symlink to it. */ + /** + * Get the collection of \ref DerivedPathWithInfo "derived paths + * with info" that this \ref Installable instalallable denotes. + * + * This is the main method of this class + */ + virtual DerivedPathsWithInfo toDerivedPaths() = 0; + + /** + * A convenience wrapper of the above for when we expect an + * installable to produce a single \ref DerivedPath "derived path" + * only. + * + * If no or multiple \ref DerivedPath "derived paths" are produced, + * and error is raised. + */ + DerivedPathWithInfo toDerivedPath(); + + /** + * Return a value only if this installable is a store path or a + * symlink to it. + * + * \todo should we move this to InstallableDerivedPath? It is only + * supposed to work there anyways. Can always downcast. + */ virtual std::optional getStorePath() { return {}; } - virtual std::vector> - getCursors(EvalState & state); - - virtual ref - getCursor(EvalState & state); - - virtual FlakeRef nixpkgsFlakeRef() const - { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); - } - - static BuiltPaths build( + static std::vector build( ref evalStore, ref store, Realise mode, - const std::vector> & installables, + const Installables & installables, BuildMode bMode = bmNormal); - static std::vector, BuiltPath>> build2( + static std::vector, BuiltPathWithResult>> build2( ref evalStore, ref store, Realise mode, - const std::vector> & installables, + const Installables & installables, BuildMode bMode = bmNormal); static std::set toStorePaths( @@ -110,18 +170,18 @@ ref store, Realise mode, OperateOn operateOn, - const std::vector> & installables); + const Installables & installables); static StorePath toStorePath( ref evalStore, ref store, Realise mode, OperateOn operateOn, - std::shared_ptr installable); + ref installable); static std::set toDerivations( ref store, - const std::vector> & installables, + const Installables & installables, bool useDeriver = false); static BuiltPaths toBuiltPaths( @@ -129,73 +189,7 @@ ref store, Realise mode, OperateOn operateOn, - const std::vector> & installables); -}; - -struct InstallableValue : Installable -{ - ref state; - - InstallableValue(ref state) : state(state) {} - - struct DerivationInfo - { - StorePath drvPath; - std::string outputName; - }; - - virtual std::vector toDerivations() = 0; - - DerivedPaths toDerivedPaths() override; - - StorePathSet toDrvPaths(ref store) override; -}; - -struct InstallableFlake : InstallableValue -{ - FlakeRef flakeRef; - Strings attrPaths; - Strings prefixes; - const flake::LockFlags & lockFlags; - mutable std::shared_ptr _lockedFlake; - - InstallableFlake( - SourceExprCommand * cmd, - ref state, - FlakeRef && flakeRef, - std::string_view fragment, - Strings attrPaths, - Strings prefixes, - const flake::LockFlags & lockFlags); - - std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } - - std::vector getActualAttrPaths(); - - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - - std::tuple toDerivation(); - - std::vector toDerivations() override; - - std::pair toValue(EvalState & state) override; - - /* Get a cursor to every attrpath in getActualAttrPaths() that - exists. */ - std::vector> - getCursors(EvalState & state) override; - - /* Get a cursor to the first attrpath in getActualAttrPaths() that - exists, or throw an exception with suggestions if none exists. */ - ref getCursor(EvalState & state) override; - - std::shared_ptr getLockedFlake() const; - - FlakeRef nixpkgsFlakeRef() const override; + const Installables & installables); }; -ref openEvalCache( - EvalState & state, - std::shared_ptr lockedFlake); - } diff -Nru nix-2.8.0/src/libcmd/installable-value.cc nix-2.16.1+dfsg/src/libcmd/installable-value.cc --- nix-2.8.0/src/libcmd/installable-value.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/installable-value.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,66 @@ +#include "installable-value.hh" +#include "eval-cache.hh" + +namespace nix { + +std::vector> +InstallableValue::getCursors(EvalState & state) +{ + auto evalCache = + std::make_shared(std::nullopt, state, + [&]() { return toValue(state).first; }); + return {evalCache->getRoot()}; +} + +ref +InstallableValue::getCursor(EvalState & state) +{ + /* Although getCursors should return at least one element, in case it doesn't, + bound check to avoid an undefined behavior for vector[0] */ + return getCursors(state).at(0); +} + +static UsageError nonValueInstallable(Installable & installable) +{ + return UsageError("installable '%s' does not correspond to a Nix language value", installable.what()); +} + +InstallableValue & InstallableValue::require(Installable & installable) +{ + auto * castedInstallable = dynamic_cast(&installable); + if (!castedInstallable) + throw nonValueInstallable(installable); + return *castedInstallable; +} + +ref InstallableValue::require(ref installable) +{ + auto castedInstallable = installable.dynamic_pointer_cast(); + if (!castedInstallable) + throw nonValueInstallable(*installable); + return ref { castedInstallable }; +} + +std::optional InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx) +{ + if (v.type() == nPath) { + auto storePath = v.path().fetchToStore(state->store); + return {{ + .path = DerivedPath::Opaque { + .path = std::move(storePath), + }, + .info = make_ref(), + }}; + } + + else if (v.type() == nString) { + return {{ + .path = state->coerceToDerivedPath(pos, v, errorCtx), + .info = make_ref(), + }}; + } + + else return std::nullopt; +} + +} diff -Nru nix-2.8.0/src/libcmd/installable-value.hh nix-2.16.1+dfsg/src/libcmd/installable-value.hh --- nix-2.8.0/src/libcmd/installable-value.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/installable-value.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,121 @@ +#pragma once +///@file + +#include "installables.hh" +#include "flake/flake.hh" + +namespace nix { + +struct DrvInfo; +struct SourceExprCommand; + +namespace eval_cache { class EvalCache; class AttrCursor; } + +struct App +{ + std::vector context; + Path program; + // FIXME: add args, sandbox settings, metadata, ... +}; + +struct UnresolvedApp +{ + App unresolved; + App resolve(ref evalStore, ref store); +}; + +/** + * Extra info about a \ref DerivedPath "derived path" that ultimately + * come from a Nix language value. + * + * Invariant: every ExtraPathInfo gotten from an InstallableValue should + * be possible to downcast to an ExtraPathInfoValue. + */ +struct ExtraPathInfoValue : ExtraPathInfo +{ + /** + * Extra struct to get around C++ designated initializer limitations + */ + struct Value { + /** + * An optional priority for use with "build envs". See Package + */ + std::optional priority; + + /** + * The attribute path associated with this value. The idea is + * that an installable referring to a value typically refers to + * a larger value, from which we project a smaller value out + * with this. + */ + std::string attrPath; + + /** + * \todo merge with DerivedPath's 'outputs' field? + */ + ExtendedOutputsSpec extendedOutputsSpec; + }; + + Value value; + + ExtraPathInfoValue(Value && v) + : value(v) + { } + + virtual ~ExtraPathInfoValue() = default; +}; + +/** + * An Installable which corresponds a Nix langauge value, in addition to + * a collection of \ref DerivedPath "derived paths". + */ +struct InstallableValue : Installable +{ + ref state; + + InstallableValue(ref state) : state(state) {} + + virtual ~InstallableValue() { } + + virtual std::pair toValue(EvalState & state) = 0; + + /** + * Get a cursor to each value this Installable could refer to. + * However if none exists, throw exception instead of returning + * empty vector. + */ + virtual std::vector> + getCursors(EvalState & state); + + /** + * Get the first and most preferred cursor this Installable could + * refer to, or throw an exception if none exists. + */ + virtual ref + getCursor(EvalState & state); + + UnresolvedApp toApp(EvalState & state); + + static InstallableValue & require(Installable & installable); + static ref require(ref installable); + +protected: + + /** + * Handles either a plain path, or a string with a single string + * context elem in the right format. The latter case is handled by + * `EvalState::coerceToDerivedPath()`; see it for details. + * + * @param v Value that is hopefully a string or path per the above. + * + * @param pos Position of value to aid with diagnostics. + * + * @param errorCtx Arbitrary message for use in potential error message when something is wrong with `v`. + * + * @result A derived path (with empty info, for now) if the value + * matched the above criteria. + */ + std::optional trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx); +}; + +} diff -Nru nix-2.8.0/src/libcmd/legacy.hh nix-2.16.1+dfsg/src/libcmd/legacy.hh --- nix-2.8.0/src/libcmd/legacy.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/legacy.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include #include diff -Nru nix-2.8.0/src/libcmd/local.mk nix-2.16.1+dfsg/src/libcmd/local.mk --- nix-2.8.0/src/libcmd/local.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -8,7 +8,7 @@ libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -libcmd_LDFLAGS += $(LOWDOWN_LIBS) -pthread +libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread libcmd_LIBS = libstore libutil libexpr libmain libfetchers diff -Nru nix-2.8.0/src/libcmd/markdown.cc nix-2.16.1+dfsg/src/libcmd/markdown.cc --- nix-2.8.0/src/libcmd/markdown.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/markdown.cc 2023-06-06 10:55:23.000000000 +0000 @@ -9,14 +9,16 @@ std::string renderMarkdownToTerminal(std::string_view markdown) { + int windowWidth = getWindowSize().second; + struct lowdown_opts opts { .type = LOWDOWN_TERM, .maxdepth = 20, - .cols = std::max(getWindowSize().second, (unsigned short) 80), + .cols = (size_t) std::max(windowWidth - 5, 60), .hmargin = 0, .vmargin = 0, .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES, - .oflags = 0, + .oflags = LOWDOWN_TERM_NOLINK, }; auto doc = lowdown_doc_new(&opts); diff -Nru nix-2.8.0/src/libcmd/markdown.hh nix-2.16.1+dfsg/src/libcmd/markdown.hh --- nix-2.8.0/src/libcmd/markdown.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/markdown.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "types.hh" namespace nix { diff -Nru nix-2.8.0/src/libcmd/nix-cmd.pc.in nix-2.16.1+dfsg/src/libcmd/nix-cmd.pc.in --- nix-2.8.0/src/libcmd/nix-cmd.pc.in 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/nix-cmd.pc.in 2023-06-06 10:55:23.000000000 +0000 @@ -6,4 +6,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixcmd -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff -Nru nix-2.8.0/src/libcmd/repl.cc nix-2.16.1+dfsg/src/libcmd/repl.cc --- nix-2.8.0/src/libcmd/repl.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/repl.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,1064 @@ +#include +#include +#include +#include + +#include + +#ifdef READLINE +#include +#include +#else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { +#include +} +#endif + +#include "repl.hh" + +#include "ansicolor.hh" +#include "shared.hh" +#include "eval.hh" +#include "eval-cache.hh" +#include "eval-inline.hh" +#include "attr-path.hh" +#include "store-api.hh" +#include "log-store.hh" +#include "common-eval-args.hh" +#include "get-drvs.hh" +#include "derivations.hh" +#include "globals.hh" +#include "flake/flake.hh" +#include "flake/lockfile.hh" +#include "editor-for.hh" +#include "finally.hh" +#include "markdown.hh" +#include "local-fs-store.hh" +#include "progress-bar.hh" +#include "print.hh" + +#if HAVE_BOEHMGC +#define GC_INCLUDE_NEW +#include +#endif + +namespace nix { + +struct NixRepl + : AbstractNixRepl + #if HAVE_BOEHMGC + , gc + #endif +{ + size_t debugTraceIndex; + + Strings loadedFiles; + std::function getValues; + + const static int envSize = 32768; + std::shared_ptr staticEnv; + Env * env; + int displ; + StringSet varNames; + + const Path historyFile; + + NixRepl(const Strings & searchPath, nix::ref store,ref state, + std::function getValues); + virtual ~NixRepl(); + + void mainLoop() override; + void initEnv() override; + + StringSet completePrefix(const std::string & prefix); + bool getLine(std::string & input, const std::string & prompt); + StorePath getDerivationPath(Value & v); + bool processLine(std::string line); + + void loadFile(const Path & path); + void loadFlake(const std::string & flakeRef); + void loadFiles(); + void reloadFiles(); + void addAttrsToScope(Value & attrs); + void addVarToScope(const Symbol name, Value & v); + Expr * parseString(std::string s); + void evalString(std::string s, Value & v); + void loadDebugTraceEnv(DebugTrace & dt); + + typedef std::set ValuesSeen; + std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth); + std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen); +}; + +std::string removeWhitespace(std::string s) +{ + s = chomp(s); + size_t n = s.find_first_not_of(" \n\r\t"); + if (n != std::string::npos) s = std::string(s, n); + return s; +} + + +NixRepl::NixRepl(const Strings & searchPath, nix::ref store, ref state, + std::function getValues) + : AbstractNixRepl(state) + , debugTraceIndex(0) + , getValues(getValues) + , staticEnv(new StaticEnv(false, state->staticBaseEnv.get())) + , historyFile(getDataDir() + "/nix/repl-history") +{ +} + + +NixRepl::~NixRepl() +{ + write_history(historyFile.c_str()); +} + +void runNix(Path program, const Strings & args, + const std::optional & input = {}) +{ + auto subprocessEnv = getEnv(); + subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); + + runProgram2(RunOptions { + .program = settings.nixBinDir+ "/" + program, + .args = args, + .environment = subprocessEnv, + .input = input, + }); + + return; +} + +static NixRepl * curRepl; // ugly + +static char * completionCallback(char * s, int *match) { + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto *res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto &first = *possible.begin(); + for (auto &p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) ++len; + if (len > 0) { + *match = 1; + auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char *s, char ***avp) { + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (INT_MAX / sizeof(char*))) + throw Error("too many completions"); + + int ac = 0; + char **vp = nullptr; + + auto check = [&](auto *p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **)malloc(possible.size() * sizeof(char*))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; +} + +namespace { + // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. + volatile sig_atomic_t g_signal_received = 0; + + void sigintHandler(int signo) { + g_signal_received = signo; + } +} + +static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) +{ + if (dt.isError) + out << ANSI_RED "error: " << ANSI_NORMAL; + out << dt.hint.str() << "\n"; + + // prefer direct pos, but if noPos then try the expr. + auto pos = dt.pos + ? dt.pos + : static_cast>(positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]); + + if (pos) { + out << pos; + if (auto loc = pos->getCodeLines()) { + out << "\n"; + printCodeLines(out, "", *pos, *loc); + out << "\n"; + } + } + + return out; +} + +void NixRepl::mainLoop() +{ + std::string error = ANSI_RED "error:" ANSI_NORMAL " "; + notice("Welcome to Nix " + nixVersion + ". Type :? for help.\n"); + + loadFiles(); + + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + try { + createDirs(dirOf(historyFile)); + } catch (SysError & e) { + logWarning(e.info()); + } +#ifndef READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); + auto oldRepl = curRepl; + curRepl = this; + Finally restoreRepl([&] { curRepl = oldRepl; }); +#ifndef READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif + + /* Stop the progress bar because it interferes with the display of + the repl. */ + stopProgressBar(); + + std::string input; + + while (true) { + // When continuing input from previous lines, don't print a prompt, just align to the same + // number of chars as the prompt. + if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { + // ctrl-D should exit the debugger. + state->debugStop = false; + state->debugQuit = true; + logger->cout(""); + break; + } + try { + if (!removeWhitespace(input).empty() && !processLine(input)) return; + } catch (ParseError & e) { + if (e.msg().find("unexpected end of file") != std::string::npos) { + // For parse errors on incomplete input, we continue waiting for the next line of + // input without clearing the input so far. + continue; + } else { + printMsg(lvlError, e.msg()); + } + } catch (EvalError & e) { + // in debugger mode, an EvalError should trigger another repl session. + // when that session returns the exception will land here. No need to show it again; + // show the error for this repl session instead. + if (state->debugRepl && !state->debugTraces.empty()) + showDebugTrace(std::cout, state->positions, state->debugTraces.front()); + else + printMsg(lvlError, e.msg()); + } catch (Error & e) { + printMsg(lvlError, e.msg()); + } catch (Interrupted & e) { + printMsg(lvlError, e.msg()); + } + + // We handled the current input fully, so we should clear it + // and read brand new input. + input.clear(); + std::cout << std::endl; + } +} + + +bool NixRepl::getLine(std::string & input, const std::string & prompt) +{ + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); + Finally resetTerminal([&]() { rl_deprep_terminal(); }); + char * s = readline(prompt.c_str()); + Finally doFree([&]() { free(s); }); + restoreSignals(); + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + + if (!s) + return false; + input += s; + input += '\n'; + return true; +} + + +StringSet NixRepl::completePrefix(const std::string & prefix) +{ + StringSet completions; + + size_t start = prefix.find_last_of(" \n\r\t(){}[]"); + std::string prev, cur; + if (start == std::string::npos) { + prev = ""; + cur = prefix; + } else { + prev = std::string(prefix, 0, start + 1); + cur = std::string(prefix, start + 1); + } + + size_t slash, dot; + + if ((slash = cur.rfind('/')) != std::string::npos) { + try { + auto dir = std::string(cur, 0, slash); + auto prefix2 = std::string(cur, slash + 1); + for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { + if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) + completions.insert(prev + dir + "/" + entry.name); + } + } catch (Error &) { + } + } else if ((dot = cur.rfind('.')) == std::string::npos) { + /* This is a variable name; look it up in the current scope. */ + StringSet::iterator i = varNames.lower_bound(cur); + while (i != varNames.end()) { + if (i->substr(0, cur.size()) != cur) break; + completions.insert(prev + *i); + i++; + } + } else { + /* Temporarily disable the debugger, to avoid re-entering readline. */ + auto debug_repl = state->debugRepl; + state->debugRepl = nullptr; + Finally restoreDebug([&]() { state->debugRepl = debug_repl; }); + try { + /* This is an expression that should evaluate to an + attribute set. Evaluate it to get the names of the + attributes. */ + auto expr = cur.substr(0, dot); + auto cur2 = cur.substr(dot + 1); + + Expr * e = parseString(expr); + Value v; + e->eval(*state, *env, v); + state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)"); + + for (auto & i : *v.attrs) { + std::string_view name = state->symbols[i.name]; + if (name.substr(0, cur2.size()) != cur2) continue; + completions.insert(concatStrings(prev, expr, ".", name)); + } + + } catch (ParseError & e) { + // Quietly ignore parse errors. + } catch (EvalError & e) { + // Quietly ignore evaluation errors. + } catch (UndefinedVarError & e) { + // Quietly ignore undefined variable errors. + } catch (BadURL & e) { + // Quietly ignore BadURL flake-related errors. + } + } + + return completions; +} + + +// FIXME: DRY and match or use the parser +static bool isVarName(std::string_view s) +{ + if (s.size() == 0) return false; + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; + for (auto & i : s) + if (!((i >= 'a' && i <= 'z') || + (i >= 'A' && i <= 'Z') || + (i >= '0' && i <= '9') || + i == '_' || i == '-' || i == '\'')) + return false; + return true; +} + + +StorePath NixRepl::getDerivationPath(Value & v) { + auto drvInfo = getDerivation(*state, v, false); + if (!drvInfo) + throw Error("expression does not evaluate to a derivation, so I can't build it"); + auto drvPath = drvInfo->queryDrvPath(); + if (!drvPath) + throw Error("expression did not evaluate to a valid derivation (no 'drvPath' attribute)"); + if (!state->store->isValidPath(*drvPath)) + throw Error("expression evaluated to invalid derivation '%s'", state->store->printStorePath(*drvPath)); + return *drvPath; +} + +void NixRepl::loadDebugTraceEnv(DebugTrace & dt) +{ + initEnv(); + + auto se = state->getStaticEnv(dt.expr); + if (se) { + auto vm = mapStaticEnvBindings(state->symbols, *se.get(), dt.env); + + // add staticenv vars. + for (auto & [name, value] : *(vm.get())) + addVarToScope(state->symbols.create(name), *value); + } +} + +bool NixRepl::processLine(std::string line) +{ + line = trim(line); + if (line == "") return true; + + _isInterrupted = false; + + std::string command, arg; + + if (line[0] == ':') { + size_t p = line.find_first_of(" \n\r\t"); + command = line.substr(0, p); + if (p != std::string::npos) arg = removeWhitespace(line.substr(p)); + } else { + arg = line; + } + + if (command == ":?" || command == ":help") { + // FIXME: convert to Markdown, include in the 'nix repl' manpage. + std::cout + << "The following commands are available:\n" + << "\n" + << " Evaluate and print expression\n" + << " = Bind expression to variable\n" + << " :a Add attributes from resulting set to scope\n" + << " :b Build a derivation\n" + << " :bl Build a derivation, creating GC roots in the working directory\n" + << " :e Open package or function in $EDITOR\n" + << " :i Build derivation, then install result into current profile\n" + << " :l Load Nix expression and add it to scope\n" + << " :lf Load Nix flake and add it to scope\n" + << " :p Evaluate and print expression recursively\n" + << " :q Exit nix-repl\n" + << " :r Reload all files\n" + << " :sh Build dependencies of derivation, then start nix-shell\n" + << " :t Describe result of evaluation\n" + << " :u Build derivation, then start nix-shell\n" + << " :doc Show documentation of a builtin function\n" + << " :log Show logs for a derivation\n" + << " :te [bool] Enable, disable or toggle showing traces for errors\n" + ; + if (state->debugRepl) { + std::cout + << "\n" + << " Debug mode commands\n" + << " :env Show env stack\n" + << " :bt Show trace stack\n" + << " :st Show current trace\n" + << " :st Change to another trace in the stack\n" + << " :c Go until end of program, exception, or builtins.break\n" + << " :s Go one step\n" + ; + } + + } + + else if (state->debugRepl && (command == ":bt" || command == ":backtrace")) { + for (const auto & [idx, i] : enumerate(state->debugTraces)) { + std::cout << "\n" << ANSI_BLUE << idx << ANSI_NORMAL << ": "; + showDebugTrace(std::cout, state->positions, i); + } + } + + else if (state->debugRepl && (command == ":env")) { + for (const auto & [idx, i] : enumerate(state->debugTraces)) { + if (idx == debugTraceIndex) { + printEnvBindings(*state, i.expr, i.env); + break; + } + } + } + + else if (state->debugRepl && (command == ":st")) { + try { + // change the DebugTrace index. + debugTraceIndex = stoi(arg); + } catch (...) { } + + for (const auto & [idx, i] : enumerate(state->debugTraces)) { + if (idx == debugTraceIndex) { + std::cout << "\n" << ANSI_BLUE << idx << ANSI_NORMAL << ": "; + showDebugTrace(std::cout, state->positions, i); + std::cout << std::endl; + printEnvBindings(*state, i.expr, i.env); + loadDebugTraceEnv(i); + break; + } + } + } + + else if (state->debugRepl && (command == ":s" || command == ":step")) { + // set flag to stop at next DebugTrace; exit repl. + state->debugStop = true; + return false; + } + + else if (state->debugRepl && (command == ":c" || command == ":continue")) { + // set flag to run to next breakpoint or end of program; exit repl. + state->debugStop = false; + return false; + } + + else if (command == ":a" || command == ":add") { + Value v; + evalString(arg, v); + addAttrsToScope(v); + } + + else if (command == ":l" || command == ":load") { + state->resetFileCache(); + loadFile(arg); + } + + else if (command == ":lf" || command == ":load-flake") { + loadFlake(arg); + } + + else if (command == ":r" || command == ":reload") { + state->resetFileCache(); + reloadFiles(); + } + + else if (command == ":e" || command == ":edit") { + Value v; + evalString(arg, v); + + const auto [path, line] = [&] () -> std::pair { + if (v.type() == nPath || v.type() == nString) { + NixStringContext context; + auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); + return {path, 0}; + } else if (v.isLambda()) { + auto pos = state->positions[v.lambda.fun->pos]; + if (auto path = std::get_if(&pos.origin)) + return {*path, pos.line}; + else + throw Error("'%s' cannot be shown in an editor", pos); + } else { + // assume it's a derivation + return findPackageFilename(*state, v, arg); + } + }(); + + // Open in EDITOR + auto args = editorFor(path, line); + auto editor = args.front(); + args.pop_front(); + + // runProgram redirects stdout to a StringSink, + // using runProgram2 to allow editors to display their UI + runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); + + // Reload right after exiting the editor + state->resetFileCache(); + reloadFiles(); + } + + else if (command == ":t") { + Value v; + evalString(arg, v); + logger->cout(showType(v)); + } + + else if (command == ":u") { + Value v, f, result; + evalString(arg, v); + evalString("drv: (import {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f); + state->callFunction(f, v, result, PosIdx()); + + StorePath drvPath = getDerivationPath(result); + runNix("nix-shell", {state->store->printStorePath(drvPath)}); + } + + else if (command == ":b" || command == ":bl" || command == ":i" || command == ":sh" || command == ":log") { + Value v; + evalString(arg, v); + StorePath drvPath = getDerivationPath(v); + Path drvPathRaw = state->store->printStorePath(drvPath); + + if (command == ":b" || command == ":bl") { + state->store->buildPaths({ + DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All { }, + }, + }); + auto drv = state->store->readDerivation(drvPath); + logger->cout("\nThis derivation produced the following outputs:"); + for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) { + auto localStore = state->store.dynamic_pointer_cast(); + if (localStore && command == ":bl") { + std::string symlink = "repl-result-" + outputName; + localStore->addPermRoot(outputPath, absPath(symlink)); + logger->cout(" ./%s -> %s", symlink, state->store->printStorePath(outputPath)); + } else { + logger->cout(" %s -> %s", outputName, state->store->printStorePath(outputPath)); + } + } + } else if (command == ":i") { + runNix("nix-env", {"-i", drvPathRaw}); + } else if (command == ":log") { + settings.readOnlyMode = true; + Finally roModeReset([&]() { + settings.readOnlyMode = false; + }); + auto subs = getDefaultSubstituters(); + + subs.push_front(state->store); + + bool foundLog = false; + RunPager pager; + for (auto & sub : subs) { + auto * logSubP = dynamic_cast(&*sub); + if (!logSubP) { + printInfo("Skipped '%s' which does not support retrieving build logs", sub->getUri()); + continue; + } + auto & logSub = *logSubP; + + auto log = logSub.getBuildLog(drvPath); + if (log) { + printInfo("got build log for '%s' from '%s'", drvPathRaw, logSub.getUri()); + logger->writeToStdout(*log); + foundLog = true; + break; + } + } + if (!foundLog) throw Error("build log of '%s' is not available", drvPathRaw); + } else { + runNix("nix-shell", {drvPathRaw}); + } + } + + else if (command == ":p" || command == ":print") { + Value v; + evalString(arg, v); + printValue(std::cout, v, 1000000000) << std::endl; + } + + else if (command == ":q" || command == ":quit") { + state->debugStop = false; + state->debugQuit = true; + return false; + } + + else if (command == ":doc") { + Value v; + evalString(arg, v); + if (auto doc = state->getDoc(v)) { + std::string markdown; + + if (!doc->args.empty() && doc->name) { + auto args = doc->args; + for (auto & arg : args) + arg = "*" + arg + "*"; + + markdown += + "**Synopsis:** `builtins." + (std::string) (*doc->name) + "` " + + concatStringsSep(" ", args) + "\n\n"; + } + + markdown += stripIndentation(doc->doc); + + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else + throw Error("value does not have documentation"); + } + + else if (command == ":te" || command == ":trace-enable") { + if (arg == "false" || (arg == "" && loggerSettings.showTrace)) { + std::cout << "not showing error traces\n"; + loggerSettings.showTrace = false; + } else if (arg == "true" || (arg == "" && !loggerSettings.showTrace)) { + std::cout << "showing error traces\n"; + loggerSettings.showTrace = true; + } else { + throw Error("unexpected argument '%s' to %s", arg, command); + }; + } + + else if (command != "") + throw Error("unknown command '%1%'", command); + + else { + size_t p = line.find('='); + std::string name; + if (p != std::string::npos && + p < line.size() && + line[p + 1] != '=' && + isVarName(name = removeWhitespace(line.substr(0, p)))) + { + Expr * e = parseString(line.substr(p + 1)); + Value & v(*state->allocValue()); + v.mkThunk(env, e); + addVarToScope(state->symbols.create(name), v); + } else { + Value v; + evalString(line, v); + printValue(std::cout, v, 1) << std::endl; + } + } + + return true; +} + +void NixRepl::loadFile(const Path & path) +{ + loadedFiles.remove(path); + loadedFiles.push_back(path); + Value v, v2; + state->evalFile(lookupFileArg(*state, path), v); + state->autoCallFunction(*autoArgs, v, v2); + addAttrsToScope(v2); +} + +void NixRepl::loadFlake(const std::string & flakeRefS) +{ + if (flakeRefS.empty()) + throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); + + auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + if (evalSettings.pureEval && !flakeRef.input.isLocked()) + throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); + + Value v; + + flake::callFlake(*state, + flake::lockFlake(*state, flakeRef, + flake::LockFlags { + .updateLockFile = false, + .useRegistries = !evalSettings.pureEval, + .allowUnlocked = !evalSettings.pureEval, + }), + v); + addAttrsToScope(v); +} + + +void NixRepl::initEnv() +{ + env = &state->allocEnv(envSize); + env->up = &state->baseEnv; + displ = 0; + staticEnv->vars.clear(); + + varNames.clear(); + for (auto & i : state->staticBaseEnv->vars) + varNames.emplace(state->symbols[i.first]); +} + + +void NixRepl::reloadFiles() +{ + initEnv(); + + loadFiles(); +} + + +void NixRepl::loadFiles() +{ + Strings old = loadedFiles; + loadedFiles.clear(); + + for (auto & i : old) { + notice("Loading '%1%'...", i); + loadFile(i); + } + + for (auto & [i, what] : getValues()) { + notice("Loading installable '%1%'...", what); + addAttrsToScope(*i); + } +} + + +void NixRepl::addAttrsToScope(Value & attrs) +{ + state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope"); + if (displ + attrs.attrs->size() >= envSize) + throw Error("environment full; cannot add more variables"); + + for (auto & i : *attrs.attrs) { + staticEnv->vars.emplace_back(i.name, displ); + env->values[displ++] = i.value; + varNames.emplace(state->symbols[i.name]); + } + staticEnv->sort(); + staticEnv->deduplicate(); + notice("Added %1% variables.", attrs.attrs->size()); +} + + +void NixRepl::addVarToScope(const Symbol name, Value & v) +{ + if (displ >= envSize) + throw Error("environment full; cannot add more variables"); + if (auto oldVar = staticEnv->find(name); oldVar != staticEnv->vars.end()) + staticEnv->vars.erase(oldVar); + staticEnv->vars.emplace_back(name, displ); + staticEnv->sort(); + env->values[displ++] = &v; + varNames.emplace(state->symbols[name]); +} + + +Expr * NixRepl::parseString(std::string s) +{ + return state->parseExprFromString(std::move(s), state->rootPath(CanonPath::fromCwd()), staticEnv); +} + + +void NixRepl::evalString(std::string s, Value & v) +{ + Expr * e = parseString(s); + e->eval(*state, *env, v); + state->forceValue(v, [&]() { return v.determinePos(noPos); }); +} + + +std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth) +{ + ValuesSeen seen; + return printValue(str, v, maxDepth, seen); +} + + + + +// FIXME: lot of cut&paste from Nix's eval.cc. +std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen) +{ + str.flush(); + checkInterrupt(); + + state->forceValue(v, [&]() { return v.determinePos(noPos); }); + + switch (v.type()) { + + case nInt: + str << ANSI_CYAN << v.integer << ANSI_NORMAL; + break; + + case nBool: + str << ANSI_CYAN; + printLiteralBool(str, v.boolean); + str << ANSI_NORMAL; + break; + + case nString: + str << ANSI_WARNING; + printLiteralString(str, v.string.s); + str << ANSI_NORMAL; + break; + + case nPath: + str << ANSI_GREEN << v.path().to_string() << ANSI_NORMAL; // !!! escaping? + break; + + case nNull: + str << ANSI_CYAN "null" ANSI_NORMAL; + break; + + case nAttrs: { + seen.insert(&v); + + bool isDrv = state->isDerivation(v); + + if (isDrv) { + str << "«derivation "; + Bindings::iterator i = v.attrs->find(state->sDrvPath); + NixStringContext context; + if (i != v.attrs->end()) + str << state->store->printStorePath(state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation")); + else + str << "???"; + str << "»"; + } + + else if (maxDepth > 0) { + str << "{ "; + + typedef std::map Sorted; + Sorted sorted; + for (auto & i : *v.attrs) + sorted.emplace(state->symbols[i.name], i.value); + + for (auto & i : sorted) { + printAttributeName(str, i.first); + str << " = "; + if (seen.count(i.second)) + str << "«repeated»"; + else + try { + printValue(str, *i.second, maxDepth - 1, seen); + } catch (AssertionError & e) { + str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL; + } + str << "; "; + } + + str << "}"; + } else + str << "{ ... }"; + + break; + } + + case nList: + seen.insert(&v); + + str << "[ "; + if (maxDepth > 0) + for (auto elem : v.listItems()) { + if (seen.count(elem)) + str << "«repeated»"; + else + try { + printValue(str, *elem, maxDepth - 1, seen); + } catch (AssertionError & e) { + str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL; + } + str << " "; + } + else + str << "... "; + str << "]"; + break; + + case nFunction: + if (v.isLambda()) { + std::ostringstream s; + s << state->positions[v.lambda.fun->pos]; + str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; + } else if (v.isPrimOp()) { + str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; + } else if (v.isPrimOpApp()) { + str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; + } else { + abort(); + } + break; + + case nFloat: + str << v.fpoint; + break; + + case nThunk: + case nExternal: + default: + str << ANSI_RED "«unknown»" ANSI_NORMAL; + break; + } + + return str; +} + + +std::unique_ptr AbstractNixRepl::create( + const Strings & searchPath, nix::ref store, ref state, + std::function getValues) +{ + return std::make_unique( + searchPath, + openStore(), + state, + getValues + ); +} + + +void AbstractNixRepl::runSimple( + ref evalState, + const ValMap & extraEnv) +{ + auto getValues = [&]()->NixRepl::AnnotatedValues{ + NixRepl::AnnotatedValues values; + return values; + }; + const Strings & searchPath = {}; + auto repl = std::make_unique( + searchPath, + openStore(), + evalState, + getValues + ); + + repl->initEnv(); + + // add 'extra' vars. + for (auto & [name, value] : extraEnv) + repl->addVarToScope(repl->state->symbols.create(name), *value); + + repl->mainLoop(); +} + +} diff -Nru nix-2.8.0/src/libcmd/repl.hh nix-2.16.1+dfsg/src/libcmd/repl.hh --- nix-2.8.0/src/libcmd/repl.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libcmd/repl.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,40 @@ +#pragma once +///@file + +#include "eval.hh" + +#if HAVE_BOEHMGC +#define GC_INCLUDE_NEW +#include +#endif + +namespace nix { + +struct AbstractNixRepl +{ + ref state; + Bindings * autoArgs; + + AbstractNixRepl(ref state) + : state(state) + { } + + virtual ~AbstractNixRepl() + { } + + typedef std::vector> AnnotatedValues; + + static std::unique_ptr create( + const Strings & searchPath, nix::ref store, ref state, + std::function getValues); + + static void runSimple( + ref evalState, + const ValMap & extraEnv); + + virtual void initEnv() = 0; + + virtual void mainLoop() = 0; +}; + +} diff -Nru nix-2.8.0/src/libexpr/attr-path.cc nix-2.16.1+dfsg/src/libexpr/attr-path.cc --- nix-2.8.0/src/libexpr/attr-path.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/attr-path.cc 2023-06-06 10:55:23.000000000 +0000 @@ -41,13 +41,13 @@ } -std::pair findAlongAttrPath(EvalState & state, const std::string & attrPath, +std::pair findAlongAttrPath(EvalState & state, const std::string & attrPath, Bindings & autoArgs, Value & vIn) { Strings tokens = parseAttrPath(attrPath); Value * v = &vIn; - Pos pos = noPos; + PosIdx pos = noPos; for (auto & attr : tokens) { @@ -77,13 +77,13 @@ if (a == v->attrs->end()) { std::set attrNames; for (auto & attr : *v->attrs) - attrNames.insert(attr.name); + attrNames.insert(state.symbols[attr.name]); auto suggestions = Suggestions::bestMatches(attrNames, attr); throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath); } v = &*a->value; - pos = *a->pos; + pos = a->pos; } else { @@ -106,7 +106,7 @@ } -Pos findPackageFilename(EvalState & state, Value & v, std::string what) +std::pair findPackageFilename(EvalState & state, Value & v, std::string what) { Value * v2; try { @@ -118,23 +118,25 @@ // FIXME: is it possible to extract the Pos object instead of doing this // toString + parsing? - auto pos = state.forceString(*v2); + NixStringContext context; + auto path = state.coerceToPath(noPos, *v2, context, "while evaluating the 'meta.position' attribute of a derivation"); - auto colon = pos.rfind(':'); - if (colon == std::string::npos) - throw ParseError("cannot parse meta.position attribute '%s'", pos); + auto fn = path.path.abs(); + + auto fail = [fn]() { + throw ParseError("cannot parse 'meta.position' attribute '%s'", fn); + }; - std::string filename(pos, 0, colon); - unsigned int lineno; try { - lineno = std::stoi(std::string(pos, colon + 1, std::string::npos)); + auto colon = fn.rfind(':'); + if (colon == std::string::npos) fail(); + std::string filename(fn, 0, colon); + auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); + return {CanonPath(fn.substr(0, colon)), lineno}; } catch (std::invalid_argument & e) { - throw ParseError("cannot parse line number '%s'", pos); + fail(); + abort(); } - - Symbol file = state.symbols.create(filename); - - return { foFile, file, lineno, 0 }; } diff -Nru nix-2.8.0/src/libexpr/attr-path.hh nix-2.16.1+dfsg/src/libexpr/attr-path.hh --- nix-2.8.0/src/libexpr/attr-path.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/attr-path.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" @@ -10,14 +11,16 @@ MakeError(AttrPathNotFound, Error); MakeError(NoPositionInfo, Error); -std::pair findAlongAttrPath( +std::pair findAlongAttrPath( EvalState & state, const std::string & attrPath, Bindings & autoArgs, Value & vIn); -/* Heuristic to find the filename and lineno or a nix value. */ -Pos findPackageFilename(EvalState & state, Value & v, std::string what); +/** + * Heuristic to find the filename and lineno or a nix value. + */ +std::pair findPackageFilename(EvalState & state, Value & v, std::string what); std::vector parseAttrPath(EvalState & state, std::string_view s); diff -Nru nix-2.8.0/src/libexpr/attr-set.cc nix-2.16.1+dfsg/src/libexpr/attr-set.cc --- nix-2.8.0/src/libexpr/attr-set.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/attr-set.cc 2023-06-06 10:55:23.000000000 +0000 @@ -26,7 +26,7 @@ /* Create a new attribute named 'name' on an existing attribute set stored in 'vAttrs' and return the newly allocated Value which is associated with this attribute. */ -Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) +Value * EvalState::allocAttr(Value & vAttrs, Symbol name) { Value * v = allocValue(); vAttrs.attrs->push_back(Attr(name, v)); @@ -40,7 +40,7 @@ } -Value & BindingsBuilder::alloc(const Symbol & name, ptr pos) +Value & BindingsBuilder::alloc(Symbol name, PosIdx pos) { auto value = state.allocValue(); bindings->push_back(Attr(name, value, pos)); @@ -48,7 +48,7 @@ } -Value & BindingsBuilder::alloc(std::string_view name, ptr pos) +Value & BindingsBuilder::alloc(std::string_view name, PosIdx pos) { return alloc(state.symbols.create(name), pos); } diff -Nru nix-2.8.0/src/libexpr/attr-set.hh nix-2.16.1+dfsg/src/libexpr/attr-set.hh --- nix-2.8.0/src/libexpr/attr-set.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/attr-set.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "nixexpr.hh" #include "symbol-table.hh" @@ -12,36 +13,49 @@ class EvalState; struct Value; -/* Map one attribute name to its value. */ +/** + * Map one attribute name to its value. + */ struct Attr { + /* the placement of `name` and `pos` in this struct is important. + both of them are uint32 wrappers, they are next to each other + to make sure that Attr has no padding on 64 bit machines. that + way we keep Attr size at two words with no wasted space. */ Symbol name; + PosIdx pos; Value * value; - ptr pos; - Attr(Symbol name, Value * value, ptr pos = ptr(&noPos)) - : name(name), value(value), pos(pos) { }; - Attr() : pos(&noPos) { }; + Attr(Symbol name, Value * value, PosIdx pos = noPos) + : name(name), pos(pos), value(value) { }; + Attr() { }; bool operator < (const Attr & a) const { return name < a.name; } }; -/* Bindings contains all the attributes of an attribute set. It is defined - by its size and its capacity, the capacity being the number of Attr - elements allocated after this structure, while the size corresponds to - the number of elements already inserted in this structure. */ +static_assert(sizeof(Attr) == 2 * sizeof(uint32_t) + sizeof(Value *), + "performance of the evaluator is highly sensitive to the size of Attr. " + "avoid introducing any padding into Attr if at all possible, and do not " + "introduce new fields that need not be present for almost every instance."); + +/** + * Bindings contains all the attributes of an attribute set. It is defined + * by its size and its capacity, the capacity being the number of Attr + * elements allocated after this structure, while the size corresponds to + * the number of elements already inserted in this structure. + */ class Bindings { public: typedef uint32_t size_t; - ptr pos; + PosIdx pos; private: size_t size_, capacity_; Attr attrs[0]; - Bindings(size_t capacity) : pos(&noPos), size_(0), capacity_(capacity) { } + Bindings(size_t capacity) : size_(0), capacity_(capacity) { } Bindings(const Bindings & bindings) = delete; public: @@ -57,7 +71,7 @@ attrs[size_++] = attr; } - iterator find(const Symbol & name) + iterator find(Symbol name) { Attr key(name, 0); iterator i = std::lower_bound(begin(), end(), key); @@ -65,7 +79,7 @@ return end(); } - Attr * get(const Symbol & name) + Attr * get(Symbol name) { Attr key(name, 0); iterator i = std::lower_bound(begin(), end(), key); @@ -73,18 +87,6 @@ return nullptr; } - Attr & need(const Symbol & name, const Pos & pos = noPos) - { - auto a = get(name); - if (!a) - throw Error({ - .msg = hintfmt("attribute '%s' missing", name), - .errPos = pos - }); - - return *a; - } - iterator begin() { return &attrs[0]; } iterator end() { return &attrs[size_]; } @@ -97,15 +99,18 @@ size_t capacity() { return capacity_; } - /* Returns the attributes in lexicographically sorted order. */ - std::vector lexicographicOrder() const + /** + * Returns the attributes in lexicographically sorted order. + */ + std::vector lexicographicOrder(const SymbolTable & symbols) const { std::vector res; res.reserve(size_); for (size_t n = 0; n < size_; n++) res.emplace_back(&attrs[n]); - std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) { - return (const std::string &) a->name < (const std::string &) b->name; + std::sort(res.begin(), res.end(), [&](const Attr * a, const Attr * b) { + std::string_view sa = symbols[a->name], sb = symbols[b->name]; + return sa < sb; }); return res; } @@ -113,9 +118,11 @@ friend class EvalState; }; -/* A wrapper around Bindings that ensures that its always in sorted - order at the end. The only way to consume a BindingsBuilder is to - call finish(), which sorts the bindings. */ +/** + * A wrapper around Bindings that ensures that its always in sorted + * order at the end. The only way to consume a BindingsBuilder is to + * call finish(), which sorts the bindings. + */ class BindingsBuilder { Bindings * bindings; @@ -130,7 +137,7 @@ : bindings(bindings), state(state) { } - void insert(Symbol name, Value * value, ptr pos = ptr(&noPos)) + void insert(Symbol name, Value * value, PosIdx pos = noPos) { insert(Attr(name, value, pos)); } @@ -145,9 +152,9 @@ bindings->push_back(attr); } - Value & alloc(const Symbol & name, ptr pos = ptr(&noPos)); + Value & alloc(Symbol name, PosIdx pos = noPos); - Value & alloc(std::string_view name, ptr pos = ptr(&noPos)); + Value & alloc(std::string_view name, PosIdx pos = noPos); Bindings * finish() { diff -Nru nix-2.8.0/src/libexpr/eval-cache.cc nix-2.16.1+dfsg/src/libexpr/eval-cache.cc --- nix-2.8.0/src/libexpr/eval-cache.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/eval-cache.cc 2023-06-06 10:55:23.000000000 +0000 @@ -35,13 +35,19 @@ std::unique_ptr> _state; - AttrDb(const Store & cfg, const Hash & fingerprint) + SymbolTable & symbols; + + AttrDb( + const Store & cfg, + const Hash & fingerprint, + SymbolTable & symbols) : cfg(cfg) , _state(std::make_unique>()) + , symbols(symbols) { auto state(_state->lock()); - Path cacheDir = getCacheDir() + "/nix/eval-cache-v2"; + Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; createDirs(cacheDir); Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; @@ -100,7 +106,7 @@ state->insertAttribute.use() (key.first) - (key.second) + (symbols[key.second]) (AttrType::FullAttrs) (0, false).exec(); @@ -110,7 +116,7 @@ for (auto & attr : attrs) state->insertAttribute.use() (rowId) - (attr) + (symbols[attr]) (AttrType::Placeholder) (0, false).exec(); @@ -135,14 +141,14 @@ } state->insertAttributeWithContext.use() (key.first) - (key.second) + (symbols[key.second]) (AttrType::String) (s) (ctx).exec(); } else { state->insertAttribute.use() (key.first) - (key.second) + (symbols[key.second]) (AttrType::String) (s).exec(); } @@ -161,7 +167,7 @@ state->insertAttribute.use() (key.first) - (key.second) + (symbols[key.second]) (AttrType::Bool) (b ? 1 : 0).exec(); @@ -169,6 +175,42 @@ }); } + AttrId setInt( + AttrKey key, + int n) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (symbols[key.second]) + (AttrType::Int) + (n).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setListOfStrings( + AttrKey key, + const std::vector & l) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (symbols[key.second]) + (AttrType::ListOfStrings) + (concatStringsSep("\t", l)).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + AttrId setPlaceholder(AttrKey key) { return doSQLite([&]() @@ -177,7 +219,7 @@ state->insertAttribute.use() (key.first) - (key.second) + (symbols[key.second]) (AttrType::Placeholder) (0, false).exec(); @@ -193,7 +235,7 @@ state->insertAttribute.use() (key.first) - (key.second) + (symbols[key.second]) (AttrType::Missing) (0, false).exec(); @@ -209,7 +251,7 @@ state->insertAttribute.use() (key.first) - (key.second) + (symbols[key.second]) (AttrType::Misc) (0, false).exec(); @@ -225,7 +267,7 @@ state->insertAttribute.use() (key.first) - (key.second) + (symbols[key.second]) (AttrType::Failed) (0, false).exec(); @@ -233,16 +275,14 @@ }); } - std::optional> getAttr( - AttrKey key, - SymbolTable & symbols) + std::optional> getAttr(AttrKey key) { auto state(_state->lock()); - auto queryAttribute(state->queryAttribute.use()(key.first)(key.second)); + auto queryAttribute(state->queryAttribute.use()(key.first)(symbols[key.second])); if (!queryAttribute.next()) return {}; - auto rowId = (AttrType) queryAttribute.getInt(0); + auto rowId = (AttrId) queryAttribute.getInt(0); auto type = (AttrType) queryAttribute.getInt(1); switch (type) { @@ -253,18 +293,22 @@ std::vector attrs; auto queryAttributes(state->queryAttributes.use()(rowId)); while (queryAttributes.next()) - attrs.push_back(symbols.create(queryAttributes.getStr(0))); + attrs.emplace_back(symbols.create(queryAttributes.getStr(0))); return {{rowId, attrs}}; } case AttrType::String: { NixStringContext context; if (!queryAttribute.isNull(3)) for (auto & s : tokenizeString>(queryAttribute.getStr(3), ";")) - context.push_back(decodeContext(cfg, s)); + context.insert(NixStringContextElem::parse(s)); return {{rowId, string_t{queryAttribute.getStr(2), context}}}; } case AttrType::Bool: return {{rowId, queryAttribute.getInt(2) != 0}}; + case AttrType::Int: + return {{rowId, int_t{queryAttribute.getInt(2)}}}; + case AttrType::ListOfStrings: + return {{rowId, tokenizeString>(queryAttribute.getStr(2), "\t")}}; case AttrType::Missing: return {{rowId, missing_t()}}; case AttrType::Misc: @@ -277,10 +321,13 @@ } }; -static std::shared_ptr makeAttrDb(const Store & cfg, const Hash & fingerprint) +static std::shared_ptr makeAttrDb( + const Store & cfg, + const Hash & fingerprint, + SymbolTable & symbols) { try { - return std::make_shared(cfg, fingerprint); + return std::make_shared(cfg, fingerprint, symbols); } catch (SQLiteError &) { ignoreException(); return nullptr; @@ -291,7 +338,7 @@ std::optional> useCache, EvalState & state, RootLoader rootLoader) - : db(useCache ? makeAttrDb(*state.store, *useCache) : nullptr) + : db(useCache ? makeAttrDb(*state.store, *useCache, state.symbols) : nullptr) , state(state) , rootLoader(rootLoader) { @@ -327,8 +374,7 @@ if (!parent) return {0, root->state.sEpsilon}; if (!parent->first->cachedValue) { - parent->first->cachedValue = root->db->getAttr( - parent->first->getKey(), root->state.symbols); + parent->first->cachedValue = root->db->getAttr(parent->first->getKey()); assert(parent->first->cachedValue); } return {parent->first->cachedValue->first, parent->second}; @@ -339,7 +385,7 @@ if (!_value) { if (parent) { auto & vParent = parent->first->getValue(); - root->state.forceAttrs(vParent, noPos); + root->state.forceAttrs(vParent, noPos, "while searching for an attribute"); auto attr = vParent.attrs->get(parent->second); if (!attr) throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr()); @@ -369,17 +415,17 @@ std::string AttrCursor::getAttrPathStr() const { - return concatStringsSep(".", getAttrPath()); + return concatStringsSep(".", root->state.symbols.resolve(getAttrPath())); } std::string AttrCursor::getAttrPathStr(Symbol name) const { - return concatStringsSep(".", getAttrPath(name)); + return concatStringsSep(".", root->state.symbols.resolve(getAttrPath(name))); } Value & AttrCursor::forceValue() { - debug("evaluating uncached attribute %s", getAttrPathStr()); + debug("evaluating uncached attribute '%s'", getAttrPathStr()); auto & v = getValue(); @@ -396,10 +442,14 @@ if (v.type() == nString) cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; - else if (v.type() == nPath) - cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; + else if (v.type() == nPath) { + auto path = v.path().path; + cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}}; + } else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; + else if (v.type() == nInt) + cachedValue = {root->db->setInt(getKey(), v.integer), int_t{v.integer}}; else if (v.type() == nAttrs) ; // FIXME: do something? else @@ -414,31 +464,31 @@ auto attrNames = getAttrs(); std::set strAttrNames; for (auto & name : attrNames) - strAttrNames.insert(std::string(name)); + strAttrNames.insert(root->state.symbols[name]); - return Suggestions::bestMatches(strAttrNames, name); + return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]); } std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErrors) { if (root->db) { if (!cachedValue) - cachedValue = root->db->getAttr(getKey(), root->state.symbols); + cachedValue = root->db->getAttr(getKey()); if (cachedValue) { if (auto attrs = std::get_if>(&cachedValue->second)) { for (auto & attr : *attrs) if (attr == name) - return std::make_shared(root, std::make_pair(shared_from_this(), name)); + return std::make_shared(root, std::make_pair(shared_from_this(), attr)); return nullptr; } else if (std::get_if(&cachedValue->second)) { - auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols); + auto attr = root->db->getAttr({cachedValue->first, name}); if (attr) { if (std::get_if(&attr->second)) return nullptr; else if (std::get_if(&attr->second)) { if (forceErrors) - debug("reevaluating failed cached attribute '%s'"); + debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name)); else throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name)); } else @@ -459,11 +509,6 @@ return nullptr; //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); - for (auto & attr : *v.attrs) { - if (root->db) - root->db->setPlaceholder({cachedValue->first, attr.name}); - } - auto attr = v.attrs->get(name); if (!attr) { @@ -522,34 +567,45 @@ { if (root->db) { if (!cachedValue) - cachedValue = root->db->getAttr(getKey(), root->state.symbols); + cachedValue = root->db->getAttr(getKey()); if (cachedValue && !std::get_if(&cachedValue->second)) { if (auto s = std::get_if(&cachedValue->second)) { debug("using cached string attribute '%s'", getAttrPathStr()); return s->first; } else - throw TypeError("'%s' is not a string", getAttrPathStr()); + root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nString && v.type() != nPath) - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())); + root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); - return v.type() == nString ? v.string.s : v.path; + return v.type() == nString ? v.string.s : v.path().to_string(); } string_t AttrCursor::getStringWithContext() { if (root->db) { if (!cachedValue) - cachedValue = root->db->getAttr(getKey(), root->state.symbols); + cachedValue = root->db->getAttr(getKey()); if (cachedValue && !std::get_if(&cachedValue->second)) { if (auto s = std::get_if(&cachedValue->second)) { bool valid = true; for (auto & c : s->second) { - if (!root->state.store->isValidPath(c.first)) { + const StorePath & path = std::visit(overloaded { + [&](const NixStringContextElem::DrvDeep & d) -> const StorePath & { + return d.drvPath; + }, + [&](const NixStringContextElem::Built & b) -> const StorePath & { + return b.drvPath; + }, + [&](const NixStringContextElem::Opaque & o) -> const StorePath & { + return o.path; + }, + }, c.raw()); + if (!root->state.store->isValidPath(path)) { valid = false; break; } @@ -559,66 +615,125 @@ return *s; } } else - throw TypeError("'%s' is not a string", getAttrPathStr()); + root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); - if (v.type() == nString) - return {v.string.s, v.getContext(*root->state.store)}; + if (v.type() == nString) { + NixStringContext context; + copyContext(v, context); + return {v.string.s, std::move(context)}; + } else if (v.type() == nPath) - return {v.path, {}}; + return {v.path().to_string(), {}}; else - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())); + root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); } bool AttrCursor::getBool() { if (root->db) { if (!cachedValue) - cachedValue = root->db->getAttr(getKey(), root->state.symbols); + cachedValue = root->db->getAttr(getKey()); if (cachedValue && !std::get_if(&cachedValue->second)) { if (auto b = std::get_if(&cachedValue->second)) { debug("using cached Boolean attribute '%s'", getAttrPathStr()); return *b; } else - throw TypeError("'%s' is not a Boolean", getAttrPathStr()); + root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nBool) - throw TypeError("'%s' is not a Boolean", getAttrPathStr()); + root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow(); return v.boolean; } +NixInt AttrCursor::getInt() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey()); + if (cachedValue && !std::get_if(&cachedValue->second)) { + if (auto i = std::get_if(&cachedValue->second)) { + debug("using cached integer attribute '%s'", getAttrPathStr()); + return i->x; + } else + throw TypeError("'%s' is not an integer", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type() != nInt) + throw TypeError("'%s' is not an integer", getAttrPathStr()); + + return v.integer; +} + +std::vector AttrCursor::getListOfStrings() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey()); + if (cachedValue && !std::get_if(&cachedValue->second)) { + if (auto l = std::get_if>(&cachedValue->second)) { + debug("using cached list of strings attribute '%s'", getAttrPathStr()); + return *l; + } else + throw TypeError("'%s' is not a list of strings", getAttrPathStr()); + } + } + + debug("evaluating uncached attribute '%s'", getAttrPathStr()); + + auto & v = getValue(); + root->state.forceValue(v, noPos); + + if (v.type() != nList) + throw TypeError("'%s' is not a list", getAttrPathStr()); + + std::vector res; + + for (auto & elem : v.listItems()) + res.push_back(std::string(root->state.forceStringNoCtx(*elem, noPos, "while evaluating an attribute for caching"))); + + if (root->db) + cachedValue = {root->db->setListOfStrings(getKey(), res), res}; + + return res; +} + std::vector AttrCursor::getAttrs() { if (root->db) { if (!cachedValue) - cachedValue = root->db->getAttr(getKey(), root->state.symbols); + cachedValue = root->db->getAttr(getKey()); if (cachedValue && !std::get_if(&cachedValue->second)) { if (auto attrs = std::get_if>(&cachedValue->second)) { debug("using cached attrset attribute '%s'", getAttrPathStr()); return *attrs; } else - throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nAttrs) - throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); std::vector attrs; for (auto & attr : *getValue().attrs) attrs.push_back(attr.name); - std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) { - return (const std::string &) a < (const std::string &) b; + std::sort(attrs.begin(), attrs.end(), [&](Symbol a, Symbol b) { + std::string_view sa = root->state.symbols[a], sb = root->state.symbols[b]; + return sa < sb; }); if (root->db) diff -Nru nix-2.8.0/src/libexpr/eval-cache.hh nix-2.16.1+dfsg/src/libexpr/eval-cache.hh --- nix-2.8.0/src/libexpr/eval-cache.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/eval-cache.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "sync.hh" #include "hash.hh" @@ -44,12 +45,15 @@ Misc = 4, Failed = 5, Bool = 6, + ListOfStrings = 7, + Int = 8, }; struct placeholder_t {}; struct missing_t {}; struct misc_t {}; struct failed_t {}; +struct int_t { NixInt x; }; typedef uint64_t AttrId; typedef std::pair AttrKey; typedef std::pair string_t; @@ -61,7 +65,9 @@ missing_t, misc_t, failed_t, - bool + bool, + int_t, + std::vector > AttrValue; class AttrCursor : public std::enable_shared_from_this @@ -104,8 +110,10 @@ ref getAttr(std::string_view name); - /* Get an attribute along a chain of attrsets. Note that this does - not auto-call functors or functions. */ + /** + * Get an attribute along a chain of attrsets. Note that this does + * not auto-call functors or functions. + */ OrSuggestions> findAlongAttrPath(const std::vector & attrPath, bool force = false); std::string getString(); @@ -114,13 +122,19 @@ bool getBool(); + NixInt getInt(); + + std::vector getListOfStrings(); + std::vector getAttrs(); bool isDerivation(); Value & forceValue(); - /* Force creation of the .drv file in the Nix store. */ + /** + * Force creation of the .drv file in the Nix store. + */ StorePath forceDerivation(); }; diff -Nru nix-2.8.0/src/libexpr/eval.cc nix-2.16.1+dfsg/src/libexpr/eval.cc --- nix-2.8.0/src/libexpr/eval.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/eval.cc 2023-06-06 10:55:23.000000000 +0000 @@ -4,22 +4,28 @@ #include "util.hh" #include "store-api.hh" #include "derivations.hh" +#include "downstream-placeholder.hh" #include "globals.hh" #include "eval-inline.hh" #include "filetransfer.hh" -#include "json.hh" #include "function-trace.hh" +#include "profiles.hh" +#include "print.hh" #include #include +#include #include +#include #include #include #include #include #include +#include #include +#include #if HAVE_BOEHMGC @@ -34,8 +40,9 @@ #endif -namespace nix { +using json = nlohmann::json; +namespace nix { static char * allocString(size_t size) { @@ -43,7 +50,7 @@ #if HAVE_BOEHMGC t = (char *) GC_MALLOC_ATOMIC(size); #else - t = malloc(size); + t = (char *) malloc(size); #endif if (!t) throw std::bad_alloc(); return t; @@ -65,26 +72,19 @@ // When there's no need to write to the string, we can optimize away empty // string allocations. -// This function handles makeImmutableStringWithLen(null, 0) by returning the -// empty string. -static const char * makeImmutableStringWithLen(const char * s, size_t size) +// This function handles makeImmutableString(std::string_view()) by returning +// the empty string. +static const char * makeImmutableString(std::string_view s) { - char * t; + const size_t size = s.size(); if (size == 0) return ""; -#if HAVE_BOEHMGC - t = GC_STRNDUP(s, size); -#else - t = strndup(s, size); -#endif - if (!t) throw std::bad_alloc(); + auto t = allocString(size + 1); + memcpy(t, s.data(), size); + t[size] = '\0'; return t; } -static inline const char * makeImmutableString(std::string_view s) { - return makeImmutableStringWithLen(s.data(), s.size()); -} - RootValue allocRootValue(Value * v) { @@ -95,8 +95,8 @@ #endif } - -void Value::print(std::ostream & str, std::set * seen) const +void Value::print(const SymbolTable & symbols, std::ostream & str, + std::set * seen) const { checkInterrupt(); @@ -105,21 +105,13 @@ str << integer; break; case tBool: - str << (boolean ? "true" : "false"); + printLiteralBool(str, boolean); break; case tString: - str << "\""; - for (const char * i = string.s; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; - else str << *i; - str << "\""; + printLiteralString(str, string.s); break; case tPath: - str << path; // !!! escaping? + str << path().to_string(); // !!! escaping? break; case tNull: str << "null"; @@ -129,9 +121,9 @@ str << "«repeated»"; else { str << "{ "; - for (auto & i : attrs->lexicographicOrder()) { - str << i->name << " = "; - i->value->print(str, seen); + for (auto & i : attrs->lexicographicOrder(symbols)) { + str << symbols[i->name] << " = "; + i->value->print(symbols, str, seen); str << "; "; } str << "}"; @@ -146,7 +138,10 @@ else { str << "[ "; for (auto v2 : listItems()) { - v2->print(str, seen); + if (v2) + v2->print(symbols, str, seen); + else + str << "(nullptr)"; str << " "; } str << "]"; @@ -171,23 +166,39 @@ case tFloat: str << fpoint; break; + case tBlackhole: + // Although we know for sure that it's going to be an infinite recursion + // when this value is accessed _in the current context_, it's likely + // that the user will misinterpret a simpler «infinite recursion» output + // as a definitive statement about the value, while in fact it may be + // a valid value after `builtins.trace` and perhaps some other steps + // have completed. + str << "«potential infinite recursion»"; + break; default: + printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType); abort(); } } -void Value::print(std::ostream & str, bool showRepeated) const +void Value::print(const SymbolTable & symbols, std::ostream & str, bool showRepeated) const { std::set seen; - print(str, showRepeated ? nullptr : &seen); + print(symbols, str, showRepeated ? nullptr : &seen); } +// Pretty print types for assertion errors +std::ostream & operator << (std::ostream & os, const ValueType t) { + os << showType(t); + return os; +} -std::ostream & operator << (std::ostream & str, const Value & v) +std::string printValue(const EvalState & state, const Value & v) { - v.print(str, false); - return str; + std::ostringstream out; + v.print(state.symbols, out); + return out.str(); } @@ -221,6 +232,9 @@ std::string showType(const Value & v) { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (v.internalType) { case tString: return v.string.context ? "a string with context" : "a string"; case tPrimOp: @@ -234,16 +248,21 @@ default: return std::string(showType(v.type())); } + #pragma GCC diagnostic pop } -Pos Value::determinePos(const Pos & pos) const +PosIdx Value::determinePos(const PosIdx pos) const { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (internalType) { - case tAttrs: return *attrs->pos; + case tAttrs: return attrs->pos; case tLambda: return lambda.fun->pos; case tApp: return app.left->determinePos(pos); default: return pos; } + #pragma GCC diagnostic pop } bool Value::isTrivial() const @@ -308,16 +327,32 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) { - if (name.symbol.set()) { + if (name.symbol) { return name.symbol; } else { Value nameValue; name.expr->eval(state, env, nameValue); - state.forceStringNoCtx(nameValue); + state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name"); return state.symbols.create(nameValue.string.s); } } +#if HAVE_BOEHMGC +/* Disable GC while this object lives. Used by CoroutineContext. + * + * Boehm keeps a count of GC_disable() and GC_enable() calls, + * and only enables GC when the count matches. + */ +class BoehmDisableGC { +public: + BoehmDisableGC() { + GC_disable(); + }; + ~BoehmDisableGC() { + GC_enable(); + }; +}; +#endif static bool gcInitialised = false; @@ -342,6 +377,15 @@ StackAllocator::defaultAllocator = &boehmGCStackAllocator; + +#if NIX_BOEHM_PATCH_VERSION != 1 + printTalkative("Unpatched BoehmGC, disabling GC inside coroutines"); + /* Used to disable GC when entering coroutines on macOS */ + create_coro_gc_hook = []() -> std::shared_ptr { + return std::make_shared(); + }; +#endif + /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases we don't need to garbage collect at all. (Collection has a @@ -361,7 +405,7 @@ size = (pageSize * pages) / 4; // 25% of RAM if (size > maxSize) size = maxSize; #endif - debug(format("setting initial heap size to %1% bytes") % size); + debug("setting initial heap size to %1% bytes", size); GC_expand_hp(size); } @@ -394,7 +438,8 @@ } if (*p == ':') { - if (isUri(std::string(start2, s.end()))) { + auto prefix = std::string(start2, s.end()); + if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) { ++p; while (p != s.end() && *p != ':') ++p; } @@ -408,6 +453,44 @@ return res; } +ErrorBuilder & ErrorBuilder::atPos(PosIdx pos) +{ + info.errPos = state.positions[pos]; + return *this; +} + +ErrorBuilder & ErrorBuilder::withTrace(PosIdx pos, const std::string_view text) +{ + info.traces.push_front(Trace{ .pos = state.positions[pos], .hint = hintformat(std::string(text)), .frame = false }); + return *this; +} + +ErrorBuilder & ErrorBuilder::withFrameTrace(PosIdx pos, const std::string_view text) +{ + info.traces.push_front(Trace{ .pos = state.positions[pos], .hint = hintformat(std::string(text)), .frame = true }); + return *this; +} + +ErrorBuilder & ErrorBuilder::withSuggestions(Suggestions & s) +{ + info.suggestions = s; + return *this; +} + +ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr) +{ + // NOTE: This is abusing side-effects. + // TODO: check compatibility with nested debugger calls. + state.debugTraces.push_front(DebugTrace { + .pos = nullptr, + .expr = expr, + .env = env, + .hint = hintformat("Fake frame for debugging purposes"), + .isError = true + }); + return *this; +} + EvalState::EvalState( const Strings & _searchPath, @@ -449,20 +532,23 @@ , sKey(symbols.create("key")) , sPath(symbols.create("path")) , sPrefix(symbols.create("prefix")) + , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) + , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix"))) , store(store) , buildStore(buildStore ? buildStore : store) + , debugRepl(nullptr) + , debugStop(false) + , debugQuit(false) + , trylevel(0) , regexCache(makeRegexCache()) #if HAVE_BOEHMGC , valueAllocCache(std::allocate_shared(traceable_allocator(), nullptr)) , env1AllocCache(std::allocate_shared(traceable_allocator(), nullptr)) -#else - , valueAllocCache(std::make_shared(nullptr)) - , env1AllocCache(std::make_shared(nullptr)) #endif , baseEnv(allocEnv(128)) - , staticBaseEnv(false, 0) + , staticBaseEnv{std::make_shared(false, nullptr)} { countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; @@ -520,19 +606,18 @@ allowedPaths->insert(store->toRealPath(storePath)); } -void EvalState::allowAndSetStorePathString(const StorePath &storePath, Value & v) +void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v) { allowPath(storePath); - auto path = store->printStorePath(storePath); - v.mkString(path, PathSet({path})); + mkStorePathString(storePath, v); } -Path EvalState::checkSourcePath(const Path & path_) +SourcePath EvalState::checkSourcePath(const SourcePath & path_) { if (!allowedPaths) return path_; - auto i = resolvedPaths.find(path_); + auto i = resolvedPaths.find(path_.path.abs()); if (i != resolvedPaths.end()) return i->second; @@ -542,9 +627,9 @@ * attacker can't append ../../... to a path that would be in allowedPaths * and thus leak symlink targets. */ - Path abspath = canonPath(path_); + Path abspath = canonPath(path_.path.abs()); - if (hasPrefix(abspath, corepkgsPrefix)) return abspath; + if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath); for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { @@ -561,12 +646,12 @@ } /* Resolve symlinks. */ - debug(format("checking access to '%s'") % abspath); - Path path = canonPath(abspath, true); + debug("checking access to '%s'", abspath); + SourcePath path = CanonPath(canonPath(abspath, true)); for (auto & i : *allowedPaths) { - if (isDirOrInDir(path, i)) { - resolvedPaths[path_] = path; + if (isDirOrInDir(path.path.abs(), i)) { + resolvedPaths.insert_or_assign(path_.path.abs(), path); return path; } } @@ -594,12 +679,12 @@ /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - checkSourcePath(uri); + checkSourcePath(CanonPath(uri)); return; } if (hasPrefix(uri, "file://")) { - checkSourcePath(std::string(uri, 7)); + checkSourcePath(CanonPath(std::string(uri, 7))); return; } @@ -607,7 +692,7 @@ } -Path EvalState::toRealPath(const Path & path, const PathSet & context) +Path EvalState::toRealPath(const Path & path, const NixStringContext & context) { // FIXME: check whether 'path' is in 'context'. return @@ -628,7 +713,7 @@ void EvalState::addConstant(const std::string & name, Value * v) { - staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl); + staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name; baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); @@ -638,25 +723,7 @@ Value * EvalState::addPrimOp(const std::string & name, size_t arity, PrimOpFun primOp) { - auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name; - Symbol sym = symbols.create(name2); - - /* Hack to make constants lazy: turn them into a application of - the primop to a dummy value. */ - if (arity == 0) { - auto vPrimOp = allocValue(); - vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = sym }); - Value v; - v.mkApp(vPrimOp, vPrimOp); - return addConstant(name, v); - } - - Value * v = allocValue(); - v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym }); - staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl); - baseEnv.values[baseEnvDispl++] = v; - baseEnv.values[0]->attrs->push_back(Attr(sym, v)); - return v; + return addPrimOp(PrimOp { .fun = primOp, .arity = arity, .name = name }); } @@ -667,21 +734,21 @@ if (primOp.arity == 0) { primOp.arity = 1; auto vPrimOp = allocValue(); - vPrimOp->mkPrimOp(new PrimOp(std::move(primOp))); + vPrimOp->mkPrimOp(new PrimOp(primOp)); Value v; v.mkApp(vPrimOp, vPrimOp); return addConstant(primOp.name, v); } - Symbol envName = primOp.name; + auto envName = symbols.create(primOp.name); if (hasPrefix(primOp.name, "__")) - primOp.name = symbols.create(std::string(primOp.name, 2)); + primOp.name = primOp.name.substr(2); Value * v = allocValue(); - v->mkPrimOp(new PrimOp(std::move(primOp))); - staticBaseEnv.vars.emplace_back(envName, baseEnvDispl); + v->mkPrimOp(new PrimOp(primOp)); + staticBaseEnv->vars.emplace_back(envName, baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; - baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v)); + baseEnv.values[0]->attrs->push_back(Attr(symbols.create(primOp.name), v)); return v; } @@ -698,7 +765,7 @@ auto v2 = &v; if (v2->primOp->doc) return Doc { - .pos = noPos, + .pos = {}, .name = v2->primOp->name, .arity = v2->primOp->arity, .args = v2->primOp->args, @@ -709,155 +776,202 @@ } -/* Every "format" object (even temporary) takes up a few hundred bytes - of stack space, which is a real killer in the recursive - evaluator. So here are some helper functions for throwing - exceptions. */ - -LocalNoInlineNoReturn(void throwEvalError(const char * s, const std::string & s2)) +// just for the current level of StaticEnv, not the whole chain. +void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) { - throw EvalError(s, s2); + std::cout << ANSI_MAGENTA; + for (auto & i : se.vars) + std::cout << st[i.first] << " "; + std::cout << ANSI_NORMAL; + std::cout << std::endl; } -LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const Suggestions & suggestions, const char * s, const std::string & s2)) +// just for the current level of Env, not the whole chain. +void printWithBindings(const SymbolTable & st, const Env & env) { - throw EvalError(ErrorInfo { - .msg = hintfmt(s, s2), - .errPos = pos, - .suggestions = suggestions, - }); + if (env.type == Env::HasWithAttrs) { + std::cout << "with: "; + std::cout << ANSI_MAGENTA; + Bindings::iterator j = env.values[0]->attrs->begin(); + while (j != env.values[0]->attrs->end()) { + std::cout << st[j->name] << " "; + ++j; + } + std::cout << ANSI_NORMAL; + std::cout << std::endl; + } } -LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const std::string & s2)) +void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env, int lvl) { - throw EvalError(ErrorInfo { - .msg = hintfmt(s, s2), - .errPos = pos - }); -} + std::cout << "Env level " << lvl << std::endl; -LocalNoInlineNoReturn(void throwEvalError(const char * s, const std::string & s2, const std::string & s3)) -{ - throw EvalError(s, s2, s3); -} + if (se.up && env.up) { + std::cout << "static: "; + printStaticEnvBindings(st, se); + printWithBindings(st, env); + std::cout << std::endl; + printEnvBindings(st, *se.up, *env.up, ++lvl); + } else { + std::cout << ANSI_MAGENTA; + // for the top level, don't print the double underscore ones; + // they are in builtins. + for (auto & i : se.vars) + if (!hasPrefix(st[i.first], "__")) + std::cout << st[i.first] << " "; + std::cout << ANSI_NORMAL; + std::cout << std::endl; + printWithBindings(st, env); // probably nothing there for the top level. + std::cout << std::endl; -LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const std::string & s2, const std::string & s3)) -{ - throw EvalError({ - .msg = hintfmt(s, s2, s3), - .errPos = pos - }); + } } -LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const Symbol & sym, const Pos & p2)) -{ - // p1 is where the error occurred; p2 is a position mentioned in the message. - throw EvalError({ - .msg = hintfmt(s, sym, p2), - .errPos = p1 - }); +void printEnvBindings(const EvalState &es, const Expr & expr, const Env & env) +{ + // just print the names for now + auto se = es.getStaticEnv(expr); + if (se) + printEnvBindings(es.symbols, *se, env, 0); +} + +void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env, ValMap & vm) +{ + // add bindings for the next level up first, so that the bindings for this level + // override the higher levels. + // The top level bindings (builtins) are skipped since they are added for us by initEnv() + if (env.up && se.up) { + mapStaticEnvBindings(st, *se.up, *env.up, vm); + + if (env.type == Env::HasWithAttrs) { + // add 'with' bindings. + Bindings::iterator j = env.values[0]->attrs->begin(); + while (j != env.values[0]->attrs->end()) { + vm[st[j->name]] = j->value; + ++j; + } + } else { + // iterate through staticenv bindings and add them. + for (auto & i : se.vars) + vm[st[i.first]] = env.values[i.second]; + } + } } -LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) +std::unique_ptr mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env) { - throw TypeError({ - .msg = hintfmt(s), - .errPos = pos - }); + auto vm = std::make_unique(); + mapStaticEnvBindings(st, se, env, *vm); + return vm; } -LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2)) +void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & expr) { - throw TypeError({ - .msg = hintfmt(s, fun.showNamePos(), s2), - .errPos = pos - }); -} + // double check we've got the debugRepl function pointer. + if (!debugRepl) + return; -LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const Suggestions & suggestions, const char * s, const ExprLambda & fun, const Symbol & s2)) -{ - throw TypeError(ErrorInfo { - .msg = hintfmt(s, fun.showNamePos(), s2), - .errPos = pos, - .suggestions = suggestions, - }); -} + auto dts = + error && expr.getPos() + ? std::make_unique( + *this, + DebugTrace { + .pos = error->info().errPos ? error->info().errPos : static_cast>(positions[expr.getPos()]), + .expr = expr, + .env = env, + .hint = error->info().msg, + .isError = true + }) + : nullptr; + if (error) + { + printError("%s\n\n", error->what()); -LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) -{ - throw TypeError(s, showType(v)); -} + if (trylevel > 0 && error->info().level != lvlInfo) + printError("This exception occurred in a 'tryEval' call. Use " ANSI_GREEN "--ignore-try" ANSI_NORMAL " to skip these.\n"); -LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const std::string & s1)) -{ - throw AssertionError({ - .msg = hintfmt(s, s1), - .errPos = pos - }); + printError(ANSI_BOLD "Starting REPL to allow you to inspect the current state of the evaluator.\n" ANSI_NORMAL); + } + + auto se = getStaticEnv(expr); + if (se) { + auto vm = mapStaticEnvBindings(symbols, *se.get(), env); + (debugRepl)(ref(shared_from_this()), *vm); + } } -LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const std::string & s1)) +void EvalState::addErrorTrace(Error & e, const char * s, const std::string & s2) const { - throw UndefinedVarError({ - .msg = hintfmt(s, s1), - .errPos = pos - }); + e.addTrace(nullptr, s, s2); } -LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const std::string & s1)) +void EvalState::addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame) const { - throw MissingArgumentError({ - .msg = hintfmt(s, s1), - .errPos = pos - }); + e.addTrace(positions[pos], hintfmt(s, s2), frame); } -LocalNoInline(void addErrorTrace(Error & e, const char * s, const std::string & s2)) +static std::unique_ptr makeDebugTraceStacker( + EvalState & state, + Expr & expr, + Env & env, + std::shared_ptr && pos, + const char * s, + const std::string & s2) { - e.addTrace(std::nullopt, s, s2); + return std::make_unique(state, + DebugTrace { + .pos = std::move(pos), + .expr = expr, + .env = env, + .hint = hintfmt(s, s2), + .isError = false + }); } -LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const std::string & s2)) +DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t) + : evalState(evalState) + , trace(std::move(t)) { - e.addTrace(pos, s, s2); + evalState.debugTraces.push_front(trace); + if (evalState.debugStop && evalState.debugRepl) + evalState.runDebugRepl(nullptr, trace.env, trace.expr); } - void Value::mkString(std::string_view s) { mkString(makeImmutableString(s)); } -static void copyContextToValue(Value & v, const PathSet & context) +static void copyContextToValue(Value & v, const NixStringContext & context) { if (!context.empty()) { size_t n = 0; v.string.context = (const char * *) allocBytes((context.size() + 1) * sizeof(char *)); for (auto & i : context) - v.string.context[n++] = dupString(i.c_str()); + v.string.context[n++] = dupString(i.to_string().c_str()); v.string.context[n] = 0; } } -void Value::mkString(std::string_view s, const PathSet & context) +void Value::mkString(std::string_view s, const NixStringContext & context) { mkString(s); copyContextToValue(*this, context); } -void Value::mkStringMove(const char * s, const PathSet & context) +void Value::mkStringMove(const char * s, const NixStringContext & context) { mkString(s); copyContextToValue(*this, context); } -void Value::mkPath(std::string_view s) +void Value::mkPath(const SourcePath & path) { - mkPath(makeImmutableString(s)); + mkPath(makeImmutableString(path.path.abs())); } @@ -871,22 +985,21 @@ if (env->type == Env::HasWithExpr) { if (noEval) return 0; Value * v = allocValue(); - evalAttrs(*env->up, (Expr *) env->values[0], *v); + evalAttrs(*env->up, (Expr *) env->values[0], *v, noPos, ""); env->values[0] = v; env->type = Env::HasWithAttrs; } Bindings::iterator j = env->values[0]->attrs->find(var.name); if (j != env->values[0]->attrs->end()) { - if (countCalls) attrSelects[*j->pos]++; + if (countCalls) attrSelects[j->pos]++; return j->value; } if (!env->prevWith) - throwUndefinedVarError(var.pos, "undefined variable '%1%'", var.name); + error("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow(); for (size_t l = env->prevWith; l; --l, env = env->up) ; } } - void EvalState::mkList(Value & v, size_t size) { v.mkList(size); @@ -911,19 +1024,51 @@ } -void EvalState::mkPos(Value & v, ptr pos) +void EvalState::mkPos(Value & v, PosIdx p) { - if (pos->file.set()) { + auto pos = positions[p]; + if (auto path = std::get_if(&pos.origin)) { auto attrs = buildBindings(3); - attrs.alloc(sFile).mkString(pos->file); - attrs.alloc(sLine).mkInt(pos->line); - attrs.alloc(sColumn).mkInt(pos->column); + attrs.alloc(sFile).mkString(path->path.abs()); + attrs.alloc(sLine).mkInt(pos.line); + attrs.alloc(sColumn).mkInt(pos.column); v.mkAttrs(attrs); } else v.mkNull(); } +void EvalState::mkStorePathString(const StorePath & p, Value & v) +{ + v.mkString( + store->printStorePath(p), + NixStringContext { + NixStringContextElem::Opaque { .path = p }, + }); +} + + +void EvalState::mkOutputString( + Value & value, + const StorePath & drvPath, + const std::string outputName, + std::optional optOutputPath) +{ + value.mkString( + optOutputPath + ? store->printStorePath(*std::move(optOutputPath)) + /* Downstream we would substitute this for an actual path once + we build the floating CA derivation */ + : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(), + NixStringContext { + NixStringContextElem::Built { + .drvPath = drvPath, + .output = outputName, + } + }); +} + + /* Create a thunk for the delayed computation of the given expression in the given environment. But if the expression is a variable, then look it up right away. This significantly reduces the number @@ -971,7 +1116,7 @@ } -void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) +void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial) { auto path = checkSourcePath(path_); @@ -981,7 +1126,7 @@ return; } - Path resolvedPath = resolveExprPath(path); + auto resolvedPath = resolveExprPath(path); if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) { v = i->second; return; @@ -1009,8 +1154,8 @@ void EvalState::cacheFile( - const Path & path, - const Path & resolvedPath, + const SourcePath & path, + const SourcePath & resolvedPath, Expr * e, Value & v, bool mustBeTrivial) @@ -1018,14 +1163,23 @@ fileParseCache[resolvedPath] = e; try { + auto dts = debugRepl + ? makeDebugTraceStacker( + *this, + *e, + this->baseEnv, + e->getPos() ? static_cast>(positions[e->getPos()]) : nullptr, + "while evaluating the file '%1%':", resolvedPath.to_string()) + : nullptr; + // Enforce that 'flake.nix' is a direct attrset, not a // computation. if (mustBeTrivial && !(dynamic_cast(e))) - throw EvalError("file '%s' must be an attribute set", path); + error("file '%s' must be an attribute set", path).debugThrow(); eval(e, v); } catch (Error & e) { - addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath); + addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string()); throw; } @@ -1040,31 +1194,31 @@ } -inline bool EvalState::evalBool(Env & env, Expr * e) -{ - Value v; - e->eval(*this, env, v); - if (v.type() != nBool) - throwTypeError("value is %1% while a Boolean was expected", v); - return v.boolean; -} - - -inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) +inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx) { - Value v; - e->eval(*this, env, v); - if (v.type() != nBool) - throwTypeError(pos, "value is %1% while a Boolean was expected", v); - return v.boolean; + try { + Value v; + e->eval(*this, env, v); + if (v.type() != nBool) + error("value is %1% while a Boolean was expected", showType(v)).withFrame(env, *e).debugThrow(); + return v.boolean; + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } -inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v) +inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx) { - e->eval(*this, env, v); - if (v.type() != nAttrs) - throwTypeError("value is %1% while a set was expected", v); + try { + e->eval(*this, env, v); + if (v.type() != nAttrs) + error("value is %1% while a set was expected", showType(v)).withFrame(env, *e).debugThrow(); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } @@ -1124,7 +1278,7 @@ } else vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); env2.values[displ++] = vAttr; - v.attrs->push_back(Attr(i.first, vAttr, ptr(&i.second.pos))); + v.attrs->push_back(Attr(i.first, vAttr, i.second.pos)); } /* If the rec contains an attribute called `__overrides', then @@ -1137,7 +1291,7 @@ Hence we need __overrides.) */ if (hasOverrides) { Value * vOverrides = (*v.attrs)[overrides->second.displ].value; - state.forceAttrs(*vOverrides, [&]() { return vOverrides->determinePos(noPos); }); + state.forceAttrs(*vOverrides, [&]() { return vOverrides->determinePos(noPos); }, "while evaluating the `__overrides` attribute"); Bindings * newBnds = state.allocBindings(v.attrs->capacity() + vOverrides->attrs->size()); for (auto & i : *v.attrs) newBnds->push_back(i); @@ -1156,7 +1310,7 @@ else for (auto & i : attrs) - v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), ptr(&i.second.pos))); + v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), i.second.pos)); /* Dynamic attrs apply *after* rec and __overrides. */ for (auto & i : dynamicAttrs) { @@ -1165,19 +1319,19 @@ state.forceValue(nameVal, i.pos); if (nameVal.type() == nNull) continue; - state.forceStringNoCtx(nameVal); - Symbol nameSym = state.symbols.create(nameVal.string.s); + state.forceStringNoCtx(nameVal, i.pos, "while evaluating the name of a dynamic attribute"); + auto nameSym = state.symbols.create(nameVal.string.s); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) - throwEvalError(i.pos, "dynamic attribute '%1%' already defined at %2%", nameSym, *j->pos); + state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow(); i.valueExpr->setName(nameSym); /* Keep sorted order so find can catch duplicates */ - v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), ptr(&i.pos))); + v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), i.pos)); v.attrs->sort(); // FIXME: inefficient } - v.attrs->pos = ptr(&pos); + v.attrs->pos = pos; } @@ -1222,10 +1376,12 @@ for (auto & i : attrPath) { if (!first) out << '.'; else first = false; try { - out << getName(i, state, env); + out << state.symbols[getName(i, state, env)]; } catch (Error & e) { - assert(!i.symbol.set()); - out << "\"${" << *i.expr << "}\""; + assert(!i.symbol); + out << "\"${"; + i.expr->show(state.symbols, out); + out << "}\""; } } return out.str(); @@ -1235,17 +1391,26 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) { Value vTmp; - ptr pos2(&noPos); + PosIdx pos2; Value * vAttrs = &vTmp; e->eval(state, env, vTmp); try { + auto dts = state.debugRepl + ? makeDebugTraceStacker( + state, + *this, + env, + state.positions[pos2], + "while evaluating the attribute '%1%'", + showAttrPath(state, env, attrPath)) + : nullptr; for (auto & i : attrPath) { state.nrLookups++; Bindings::iterator j; - Symbol name = getName(i, state, env); + auto name = getName(i, state, env); if (def) { state.forceValue(*vAttrs, pos); if (vAttrs->type() != nAttrs || @@ -1255,28 +1420,31 @@ return; } } else { - state.forceAttrs(*vAttrs, pos); + state.forceAttrs(*vAttrs, pos, "while selecting an attribute"); if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { std::set allAttrNames; for (auto & attr : *vAttrs->attrs) - allAttrNames.insert(attr.name); - throwEvalError( - pos, - Suggestions::bestMatches(allAttrNames, name), - "attribute '%1%' missing", name); + allAttrNames.insert(state.symbols[attr.name]); + auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); + state.error("attribute '%1%' missing", state.symbols[name]) + .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow(); } } vAttrs = j->value; pos2 = j->pos; - if (state.countCalls) state.attrSelects[*pos2]++; + if (state.countCalls) state.attrSelects[pos2]++; } - state.forceValue(*vAttrs, (*pos2 != noPos ? *pos2 : this->pos ) ); + state.forceValue(*vAttrs, (pos2 ? pos2 : this->pos ) ); } catch (Error & e) { - if (*pos2 != noPos && pos2->file != state.sDerivationNix) - addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'", - showAttrPath(state, env, attrPath)); + if (pos2) { + auto pos2r = state.positions[pos2]; + auto origin = std::get_if(&pos2r.origin); + if (!(origin && *origin == state.derivationInternal)) + state.addErrorTrace(e, pos2, "while evaluating the attribute '%1%'", + showAttrPath(state, env, attrPath)); + } throw; } @@ -1294,7 +1462,7 @@ for (auto & i : attrPath) { state.forceValue(*vAttrs, noPos); Bindings::iterator j; - Symbol name = getName(i, state, env); + auto name = getName(i, state, env); if (vAttrs->type() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { @@ -1315,9 +1483,11 @@ } -void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos) +void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos) { - auto trace = evalSettings.traceFunctionCalls ? std::make_unique(pos) : nullptr; + auto trace = evalSettings.traceFunctionCalls + ? std::make_unique(positions[pos]) + : nullptr; forceValue(fun, pos); @@ -1342,7 +1512,7 @@ ExprLambda & lambda(*vCur.lambda.fun); auto size = - (lambda.arg.empty() ? 0 : 1) + + (!lambda.arg ? 0 : 1) + (lambda.hasFormals() ? lambda.formals->formals.size() : 0); Env & env2(allocEnv(size)); env2.up = vCur.lambda.env; @@ -1351,11 +1521,15 @@ if (!lambda.hasFormals()) env2.values[displ++] = args[0]; - else { - forceAttrs(*args[0], pos); + try { + forceAttrs(*args[0], lambda.pos, "while evaluating the value passed for the lambda argument"); + } catch (Error & e) { + if (pos) e.addTrace(positions[pos], "from call site"); + throw; + } - if (!lambda.arg.empty()) + if (lambda.arg) env2.values[displ++] = args[0]; /* For each formal argument, get the actual argument. If @@ -1365,8 +1539,15 @@ for (auto & i : lambda.formals->formals) { auto j = args[0]->attrs->get(i.name); if (!j) { - if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", - lambda, i.name); + if (!i.def) { + error("function '%1%' called without required argument '%2%'", + (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), + symbols[i.name]) + .atPos(lambda.pos) + .withTrace(pos, "from call site") + .withFrame(*fun.lambda.env, lambda) + .debugThrow(); + } env2.values[displ++] = i.def->maybeThunk(*this, env2); } else { attrsUsed++; @@ -1383,13 +1564,16 @@ if (!lambda.formals->has(i.name)) { std::set formalNames; for (auto & formal : lambda.formals->formals) - formalNames.insert(formal.name); - throwTypeError( - pos, - Suggestions::bestMatches(formalNames, i.name), - "%1% called with unexpected argument '%2%'", - lambda, - i.name); + formalNames.insert(symbols[formal.name]); + auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]); + error("function '%1%' called with unexpected argument '%2%'", + (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), + symbols[i.name]) + .atPos(lambda.pos) + .withTrace(pos, "from call site") + .withSuggestions(suggestions) + .withFrame(*fun.lambda.env, lambda) + .debugThrow(); } abort(); // can't happen } @@ -1400,14 +1584,27 @@ /* Evaluate the body. */ try { + auto dts = debugRepl + ? makeDebugTraceStacker( + *this, *lambda.body, env2, positions[lambda.pos], + "while calling %s", + lambda.name + ? concatStrings("'", symbols[lambda.name], "'") + : "anonymous lambda") + : nullptr; + lambda.body->eval(*this, env2, vCur); } catch (Error & e) { if (loggerSettings.showTrace.get()) { - addErrorTrace(e, lambda.pos, "while evaluating %s", - (lambda.name.set() - ? "'" + (const std::string &) lambda.name + "'" - : "anonymous lambda")); - addErrorTrace(e, pos, "from call site%s", ""); + addErrorTrace( + e, + lambda.pos, + "while calling %s", + lambda.name + ? concatStrings("'", symbols[lambda.name], "'") + : "anonymous lambda", + true); + if (pos) addErrorTrace(e, pos, "from call site%s", "", true); } throw; } @@ -1426,9 +1623,17 @@ return; } else { /* We have all the arguments, so call the primop. */ + auto name = vCur.primOp->name; + nrPrimOpCalls++; - if (countCalls) primOpCalls[vCur.primOp->name]++; - vCur.primOp->fun(*this, pos, args, vCur); + if (countCalls) primOpCalls[name]++; + + try { + vCur.primOp->fun(*this, noPos, args, vCur); + } catch (Error & e) { + addErrorTrace(e, pos, "while calling the '%1%' builtin", name); + throw; + } nrArgs -= argsLeft; args += argsLeft; @@ -1463,9 +1668,20 @@ for (size_t i = 0; i < argsLeft; ++i) vArgs[argsDone + i] = args[i]; + auto name = primOp->primOp->name; nrPrimOpCalls++; - if (countCalls) primOpCalls[primOp->primOp->name]++; - primOp->primOp->fun(*this, pos, vArgs, vCur); + if (countCalls) primOpCalls[name]++; + + try { + // TODO: + // 1. Unify this and above code. Heavily redundant. + // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc) + // so the debugger allows to inspect the wrong parameters passed to the builtin. + primOp->primOp->fun(*this, noPos, vArgs, vCur); + } catch (Error & e) { + addErrorTrace(e, pos, "while calling the '%1%' builtin", name); + throw; + } nrArgs -= argsLeft; args += argsLeft; @@ -1478,14 +1694,18 @@ heap-allocate a copy and use that instead. */ Value * args2[] = {allocValue(), args[0]}; *args2[0] = vCur; - /* !!! Should we use the attr pos here? */ - callFunction(*functor->value, 2, args2, vCur, pos); + try { + callFunction(*functor->value, 2, args2, vCur, functor->pos); + } catch (Error & e) { + e.addTrace(positions[pos], "while calling a functor (an attribute set with a '__functor' attribute)"); + throw; + } nrArgs--; args++; } else - throwTypeError(pos, "attempt to call something which is not a function but %1%", vCur); + error("attempt to call something which is not a function but %1%", showType(vCur)).atPos(pos).debugThrow(); } vRes = vCur; @@ -1549,13 +1769,12 @@ if (j != args.end()) { attrs.insert(*j); } else if (!i.def) { - throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') - + error(R"(cannot evaluate a function that has an argument without a value ('%1%') Nix attempted to evaluate a function as a top level expression; in this case it must have its arguments supplied either by default values, or passed explicitly with '--arg' or '--argstr'. See -https://nixos.org/manual/nix/stable/#ss-functions.)", i.name); - +https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbols[i.name]) + .atPos(i.pos).withFrame(*fun.lambda.env, *fun.lambda.fun).debugThrow(); } } } @@ -1578,16 +1797,17 @@ void ExprIf::eval(EvalState & state, Env & env, Value & v) { - (state.evalBool(env, cond, pos) ? then : else_)->eval(state, env, v); + // We cheat in the parser, and pass the position of the condition as the position of the if itself. + (state.evalBool(env, cond, pos, "while evaluating a branch condition") ? then : else_)->eval(state, env, v); } void ExprAssert::eval(EvalState & state, Env & env, Value & v) { - if (!state.evalBool(env, cond, pos)) { + if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) { std::ostringstream out; - cond->show(out); - throwAssertionError(pos, "assertion '%1%' failed", out.str()); + cond->show(state.symbols, out); + state.error("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow(); } body->eval(state, env, v); } @@ -1595,7 +1815,7 @@ void ExprOpNot::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(!state.evalBool(env, e)); + v.mkBool(!state.evalBool(env, e, noPos, "in the argument of the not operator")); // XXX: FIXME: ! } @@ -1603,7 +1823,7 @@ { Value v1; e1->eval(state, env, v1); Value v2; e2->eval(state, env, v2); - v.mkBool(state.eqValues(v1, v2)); + v.mkBool(state.eqValues(v1, v2, pos, "while testing two values for equality")); } @@ -1611,33 +1831,33 @@ { Value v1; e1->eval(state, env, v1); Value v2; e2->eval(state, env, v2); - v.mkBool(!state.eqValues(v1, v2)); + v.mkBool(!state.eqValues(v1, v2, pos, "while testing two values for inequality")); } void ExprOpAnd::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos)); + v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the AND (&&) operator") && state.evalBool(env, e2, pos, "in the right operand of the AND (&&) operator")); } void ExprOpOr::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); + v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the OR (||) operator") || state.evalBool(env, e2, pos, "in the right operand of the OR (||) operator")); } void ExprOpImpl::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(!state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); + v.mkBool(!state.evalBool(env, e1, pos, "in the left operand of the IMPL (->) operator") || state.evalBool(env, e2, pos, "in the right operand of the IMPL (->) operator")); } void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) { Value v1, v2; - state.evalAttrs(env, e1, v1); - state.evalAttrs(env, e2, v2); + state.evalAttrs(env, e1, v1, pos, "in the left operand of the update (//) operator"); + state.evalAttrs(env, e2, v2, pos, "in the right operand of the update (//) operator"); state.nrOpUpdates++; @@ -1676,18 +1896,18 @@ Value v1; e1->eval(state, env, v1); Value v2; e2->eval(state, env, v2); Value * lists[2] = { &v1, &v2 }; - state.concatLists(v, 2, lists, pos); + state.concatLists(v, 2, lists, pos, "while evaluating one of the elements to concatenate"); } -void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos) +void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx) { nrListConcats++; Value * nonEmpty = 0; size_t len = 0; for (size_t n = 0; n < nrLists; ++n) { - forceList(*lists[n], pos); + forceList(*lists[n], pos, errorCtx); auto l = lists[n]->listSize(); len += l; if (l) nonEmpty = lists[n]; @@ -1711,7 +1931,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) { - PathSet context; + NixStringContext context; std::vector s; size_t sSize = 0; NixInt n = 0; @@ -1764,20 +1984,22 @@ nf = n; nf += vTmp.fpoint; } else - throwEvalError(i_pos, "cannot add %1% to an integer", showType(vTmp)); + state.error("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else if (firstType == nFloat) { if (vTmp.type() == nInt) { nf += vTmp.integer; } else if (vTmp.type() == nFloat) { nf += vTmp.fpoint; } else - throwEvalError(i_pos, "cannot add %1% to a float", showType(vTmp)); + state.error("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else { if (s.empty()) s.reserve(es->size()); /* skip canonization of first path, which would only be not canonized in the first place if it's coming from a ./${foo} type path */ - auto part = state.coerceToString(i_pos, vTmp, context, false, firstType == nString, !first); + auto part = state.coerceToString(i_pos, vTmp, context, + "while evaluating a path segment", + false, firstType == nString, !first); sSize += part->size(); s.emplace_back(std::move(part)); } @@ -1791,8 +2013,8 @@ v.mkFloat(nf); else if (firstType == nPath) { if (!context.empty()) - throwEvalError(pos, "a string that refers to a store path cannot be appended to a path"); - v.mkPath(canonPath(str())); + state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); + v.mkPath(CanonPath(canonPath(str()))); } else v.mkStringMove(c_str(), context); } @@ -1800,7 +2022,7 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v) { - state.mkPos(v, ptr(&pos)); + state.mkPos(v, pos); } @@ -1818,9 +2040,15 @@ if (v.type() == nAttrs) { for (auto & i : *v.attrs) try { + // If the value is a thunk, we're evaling. Otherwise no trace necessary. + auto dts = debugRepl && i.value->isThunk() + ? makeDebugTraceStacker(*this, *i.value->thunk.expr, *i.value->thunk.env, positions[i.pos], + "while evaluating the attribute '%1%'", symbols[i.name]) + : nullptr; + recurse(*i.value); } catch (Error & e) { - addErrorTrace(e, *i.pos, "while evaluating the attribute '%1%'", i.name); + addErrorTrace(e, i.pos, "while evaluating the attribute '%1%'", symbols[i.name]); throw; } } @@ -1835,32 +2063,47 @@ } -NixInt EvalState::forceInt(Value & v, const Pos & pos) +NixInt EvalState::forceInt(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (v.type() != nInt) - throwTypeError(pos, "value is %1% while an integer was expected", v); - return v.integer; + try { + forceValue(v, pos); + if (v.type() != nInt) + error("value is %1% while an integer was expected", showType(v)).debugThrow(); + return v.integer; + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } -NixFloat EvalState::forceFloat(Value & v, const Pos & pos) +NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (v.type() == nInt) - return v.integer; - else if (v.type() != nFloat) - throwTypeError(pos, "value is %1% while a float was expected", v); - return v.fpoint; + try { + forceValue(v, pos); + if (v.type() == nInt) + return v.integer; + else if (v.type() != nFloat) + error("value is %1% while a float was expected", showType(v)).debugThrow(); + return v.fpoint; + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } -bool EvalState::forceBool(Value & v, const Pos & pos) +bool EvalState::forceBool(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (v.type() != nBool) - throwTypeError(pos, "value is %1% while a Boolean was expected", v); - return v.boolean; + try { + forceValue(v, pos); + if (v.type() != nBool) + error("value is %1% while a Boolean was expected", showType(v)).debugThrow(); + return v.boolean; + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } @@ -1870,85 +2113,54 @@ } -void EvalState::forceFunction(Value & v, const Pos & pos) +void EvalState::forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (v.type() != nFunction && !isFunctor(v)) - throwTypeError(pos, "value is %1% while a function was expected", v); -} - - -std::string_view EvalState::forceString(Value & v, const Pos & pos) -{ - forceValue(v, pos); - if (v.type() != nString) { - if (pos) - throwTypeError(pos, "value is %1% while a string was expected", v); - else - throwTypeError("value is %1% while a string was expected", v); + try { + forceValue(v, pos); + if (v.type() != nFunction && !isFunctor(v)) + error("value is %1% while a function was expected", showType(v)).debugThrow(); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; } - return v.string.s; } -/* Decode a context string ‘!!’ into a pair . */ -NixStringContextElem decodeContext(const Store & store, std::string_view s) +std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string_view errorCtx) { - if (s.at(0) == '!') { - size_t index = s.find("!", 1); - return { - store.parseStorePath(s.substr(index + 1)), - std::string(s.substr(1, index - 1)), - }; - } else - return { - store.parseStorePath( - s.at(0) == '/' - ? s - : s.substr(1)), - "", - }; + try { + forceValue(v, pos); + if (v.type() != nString) + error("value is %1% while a string was expected", showType(v)).debugThrow(); + return v.string.s; + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } -void copyContext(const Value & v, PathSet & context) +void copyContext(const Value & v, NixStringContext & context) { if (v.string.context) for (const char * * p = v.string.context; *p; ++p) - context.insert(*p); + context.insert(NixStringContextElem::parse(*p)); } -NixStringContext Value::getContext(const Store & store) +std::string_view EvalState::forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx) { - NixStringContext res; - assert(internalType == tString); - if (string.context) - for (const char * * p = string.context; *p; ++p) - res.push_back(decodeContext(store, *p)); - return res; -} - - -std::string_view EvalState::forceString(Value & v, PathSet & context, const Pos & pos) -{ - auto s = forceString(v, pos); + auto s = forceString(v, pos, errorCtx); copyContext(v, context); return s; } -std::string_view EvalState::forceStringNoCtx(Value & v, const Pos & pos) +std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx) { - auto s = forceString(v, pos); + auto s = forceString(v, pos, errorCtx); if (v.string.context) { - if (pos) - throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')", - v.string.s, v.string.context[0]); - else - throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')", - v.string.s, v.string.context[0]); + error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string.s, v.string.context[0]).withTrace(pos, errorCtx).debugThrow(); } return s; } @@ -1959,27 +2171,35 @@ if (v.type() != nAttrs) return false; Bindings::iterator i = v.attrs->find(sType); if (i == v.attrs->end()) return false; - forceValue(*i->value, *i->pos); + forceValue(*i->value, i->pos); if (i->value->type() != nString) return false; return strcmp(i->value->string.s, "derivation") == 0; } -std::optional EvalState::tryAttrsToString(const Pos & pos, Value & v, - PathSet & context, bool coerceMore, bool copyToStore) +std::optional EvalState::tryAttrsToString(const PosIdx pos, Value & v, + NixStringContext & context, bool coerceMore, bool copyToStore) { auto i = v.attrs->find(sToString); if (i != v.attrs->end()) { Value v1; callFunction(*i->value, v, v1, pos); - return coerceToString(pos, v1, context, coerceMore, copyToStore).toOwned(); + return coerceToString(pos, v1, context, + "while evaluating the result of the `__toString` attribute", + coerceMore, copyToStore).toOwned(); } return {}; } -BackedStringView EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, - bool coerceMore, bool copyToStore, bool canonicalizePath) +BackedStringView EvalState::coerceToString( + const PosIdx pos, + Value & v, + NixStringContext & context, + std::string_view errorCtx, + bool coerceMore, + bool copyToStore, + bool canonicalizePath) { forceValue(v, pos); @@ -1989,12 +2209,14 @@ } if (v.type() == nPath) { - BackedStringView path(PathView(v.path)); - if (canonicalizePath) - path = canonPath(*path); - if (copyToStore) - path = copyPathToStore(context, std::move(path).toOwned()); - return path; + return + !canonicalizePath && !copyToStore + ? // FIXME: hack to preserve path literals that end in a + // slash, as in /foo/${x}. + v._path + : copyToStore + ? store->printStorePath(copyPathToStore(context, v.path())) + : std::string(v.path().path.abs()); } if (v.type() == nAttrs) { @@ -2002,15 +2224,25 @@ if (maybeString) return std::move(*maybeString); auto i = v.attrs->find(sOutPath); - if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string"); - return coerceToString(pos, *i->value, context, coerceMore, copyToStore); + if (i == v.attrs->end()) { + error("cannot coerce %1% to a string", showType(v)) + .withTrace(pos, errorCtx) + .debugThrow(); + } + return coerceToString(pos, *i->value, context, errorCtx, + coerceMore, copyToStore, canonicalizePath); } - if (v.type() == nExternal) - return v.external->coerceToString(pos, context, coerceMore, copyToStore); + if (v.type() == nExternal) { + try { + return v.external->coerceToString(positions[pos], context, coerceMore, copyToStore); + } catch (Error & e) { + e.addTrace(nullptr, errorCtx); + throw; + } + } if (coerceMore) { - /* Note that `false' is represented as an empty string for shell scripting convenience, just like `null'. */ if (v.type() == nBool && v.boolean) return "1"; @@ -2022,7 +2254,14 @@ if (v.isList()) { std::string result; for (auto [n, v2] : enumerate(v.listItems())) { - result += *coerceToString(pos, *v2, context, coerceMore, copyToStore); + try { + result += *coerceToString(noPos, *v2, context, + "while evaluating one element of the list", + coerceMore, copyToStore, canonicalizePath); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } if (n < v.listSize() - 1 /* !!! not quite correct */ && (!v2->isList() || v2->listSize() != 0)) @@ -2032,56 +2271,129 @@ } } - throwTypeError(pos, "cannot coerce %1% to a string", v); + error("cannot coerce %1% to a string", showType(v)) + .withTrace(pos, errorCtx) + .debugThrow(); } -std::string EvalState::copyPathToStore(PathSet & context, const Path & path) +StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path) { - if (nix::isDerivation(path)) - throwEvalError("file names are not allowed to end in '%1%'", drvExtension); + if (nix::isDerivation(path.path.abs())) + error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); - Path dstPath; auto i = srcToStore.find(path); - if (i != srcToStore.end()) - dstPath = store->printStorePath(i->second); - else { - auto p = settings.readOnlyMode - ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first - : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); - dstPath = store->printStorePath(p); - allowPath(p); - srcToStore.insert_or_assign(path, std::move(p)); - printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); - } - context.insert(dstPath); + auto dstPath = i != srcToStore.end() + ? i->second + : [&]() { + auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair); + allowPath(dstPath); + srcToStore.insert_or_assign(path, dstPath); + printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); + return dstPath; + }(); + + context.insert(NixStringContextElem::Opaque { + .path = dstPath + }); return dstPath; } -Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context) +SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { - auto path = coerceToString(pos, v, context, false, false).toOwned(); + auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') - throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path); - return path; + error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); + return CanonPath(path); } -StorePath EvalState::coerceToStorePath(const Pos & pos, Value & v, PathSet & context) +StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { - auto path = coerceToString(pos, v, context, false, false).toOwned(); + auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (auto storePath = store->maybeParseStorePath(path)) return *storePath; - throw EvalError({ - .msg = hintfmt("path '%1%' is not in the Nix store", path), - .errPos = pos - }); + error("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow(); } -bool EvalState::eqValues(Value & v1, Value & v2) +std::pair EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) +{ + NixStringContext context; + auto s = forceString(v, context, pos, errorCtx); + auto csize = context.size(); + if (csize != 1) + error( + "string '%s' has %d entries in its context. It should only have exactly one entry", + s, csize) + .withTrace(pos, errorCtx).debugThrow(); + auto derivedPath = std::visit(overloaded { + [&](NixStringContextElem::Opaque && o) -> DerivedPath { + return DerivedPath::Opaque { + .path = std::move(o.path), + }; + }, + [&](NixStringContextElem::DrvDeep &&) -> DerivedPath { + error( + "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time", + s).withTrace(pos, errorCtx).debugThrow(); + }, + [&](NixStringContextElem::Built && b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = std::move(b.drvPath), + .outputs = OutputsSpec::Names { std::move(b.output) }, + }; + }, + }, ((NixStringContextElem &&) *context.begin()).raw()); + return { + std::move(derivedPath), + std::move(s), + }; +} + + +DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) +{ + auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx); + auto s = s_; + std::visit(overloaded { + [&](const DerivedPath::Opaque & o) { + auto sExpected = store->printStorePath(o.path); + if (s != sExpected) + error( + "path string '%s' has context with the different path '%s'", + s, sExpected) + .withTrace(pos, errorCtx).debugThrow(); + }, + [&](const DerivedPath::Built & b) { + // TODO need derived path with single output to make this + // total. Will add as part of RFC 92 work and then this is + // cleaned up. + auto output = *std::get(b.outputs).begin(); + + auto drv = store->readDerivation(b.drvPath); + auto i = drv.outputs.find(output); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output); + auto optOutputPath = i->second.path(*store, drv.name, output); + // This is testing for the case of CA derivations + auto sExpected = optOutputPath + ? store->printStorePath(*optOutputPath) + : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render(); + if (s != sExpected) + error( + "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", + s, output, store->printStorePath(b.drvPath), sExpected) + .withTrace(pos, errorCtx).debugThrow(); + } + }, derivedPath.raw()); + return derivedPath; +} + + +bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) { forceValue(v1, noPos); forceValue(v2, noPos); @@ -2101,7 +2413,6 @@ if (v1.type() != v2.type()) return false; switch (v1.type()) { - case nInt: return v1.integer == v2.integer; @@ -2112,7 +2423,7 @@ return strcmp(v1.string.s, v2.string.s) == 0; case nPath: - return strcmp(v1.path, v2.path) == 0; + return strcmp(v1._path, v2._path) == 0; case nNull: return true; @@ -2120,7 +2431,7 @@ case nList: if (v1.listSize() != v2.listSize()) return false; for (size_t n = 0; n < v1.listSize(); ++n) - if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false; + if (!eqValues(*v1.listElems()[n], *v2.listElems()[n], pos, errorCtx)) return false; return true; case nAttrs: { @@ -2130,7 +2441,7 @@ Bindings::iterator i = v1.attrs->find(sOutPath); Bindings::iterator j = v2.attrs->find(sOutPath); if (i != v1.attrs->end() && j != v2.attrs->end()) - return eqValues(*i->value, *j->value); + return eqValues(*i->value, *j->value, pos, errorCtx); } if (v1.attrs->size() != v2.attrs->size()) return false; @@ -2138,7 +2449,7 @@ /* Otherwise, compare the attributes one by one. */ Bindings::iterator i, j; for (i = v1.attrs->begin(), j = v2.attrs->begin(); i != v1.attrs->end(); ++i, ++j) - if (i->name != j->name || !eqValues(*i->value, *j->value)) + if (i->name != j->name || !eqValues(*i->value, *j->value, pos, errorCtx)) return false; return true; @@ -2154,8 +2465,9 @@ case nFloat: return v1.fpoint == v2.fpoint; + case nThunk: // Must not be left by forceValue default: - throwEvalError("cannot compare %1% with %2%", showType(v1), showType(v2)); + error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow(); } } @@ -2181,107 +2493,108 @@ std::fstream fs; if (outPath != "-") fs.open(outPath, std::fstream::out); - JSONObject topObj(outPath == "-" ? std::cerr : fs, true); - topObj.attr("cpuTime",cpuTime); - { - auto envs = topObj.object("envs"); - envs.attr("number", nrEnvs); - envs.attr("elements", nrValuesInEnvs); - envs.attr("bytes", bEnvs); - } - { - auto lists = topObj.object("list"); - lists.attr("elements", nrListElems); - lists.attr("bytes", bLists); - lists.attr("concats", nrListConcats); - } - { - auto values = topObj.object("values"); - values.attr("number", nrValues); - values.attr("bytes", bValues); - } - { - auto syms = topObj.object("symbols"); - syms.attr("number", symbols.size()); - syms.attr("bytes", symbols.totalSize()); - } - { - auto sets = topObj.object("sets"); - sets.attr("number", nrAttrsets); - sets.attr("bytes", bAttrsets); - sets.attr("elements", nrAttrsInAttrsets); - } - { - auto sizes = topObj.object("sizes"); - sizes.attr("Env", sizeof(Env)); - sizes.attr("Value", sizeof(Value)); - sizes.attr("Bindings", sizeof(Bindings)); - sizes.attr("Attr", sizeof(Attr)); - } - topObj.attr("nrOpUpdates", nrOpUpdates); - topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied); - topObj.attr("nrThunks", nrThunks); - topObj.attr("nrAvoided", nrAvoided); - topObj.attr("nrLookups", nrLookups); - topObj.attr("nrPrimOpCalls", nrPrimOpCalls); - topObj.attr("nrFunctionCalls", nrFunctionCalls); + json topObj = json::object(); + topObj["cpuTime"] = cpuTime; + topObj["envs"] = { + {"number", nrEnvs}, + {"elements", nrValuesInEnvs}, + {"bytes", bEnvs}, + }; + topObj["list"] = { + {"elements", nrListElems}, + {"bytes", bLists}, + {"concats", nrListConcats}, + }; + topObj["values"] = { + {"number", nrValues}, + {"bytes", bValues}, + }; + topObj["symbols"] = { + {"number", symbols.size()}, + {"bytes", symbols.totalSize()}, + }; + topObj["sets"] = { + {"number", nrAttrsets}, + {"bytes", bAttrsets}, + {"elements", nrAttrsInAttrsets}, + }; + topObj["sizes"] = { + {"Env", sizeof(Env)}, + {"Value", sizeof(Value)}, + {"Bindings", sizeof(Bindings)}, + {"Attr", sizeof(Attr)}, + }; + topObj["nrOpUpdates"] = nrOpUpdates; + topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied; + topObj["nrThunks"] = nrThunks; + topObj["nrAvoided"] = nrAvoided; + topObj["nrLookups"] = nrLookups; + topObj["nrPrimOpCalls"] = nrPrimOpCalls; + topObj["nrFunctionCalls"] = nrFunctionCalls; #if HAVE_BOEHMGC - { - auto gc = topObj.object("gc"); - gc.attr("heapSize", heapSize); - gc.attr("totalBytes", totalBytes); - } + topObj["gc"] = { + {"heapSize", heapSize}, + {"totalBytes", totalBytes}, + }; #endif if (countCalls) { + topObj["primops"] = primOpCalls; { - auto obj = topObj.object("primops"); - for (auto & i : primOpCalls) - obj.attr(i.first, i.second); - } - { - auto list = topObj.list("functions"); - for (auto & i : functionCalls) { - auto obj = list.object(); - if (i.first->name.set()) - obj.attr("name", (const std::string &) i.first->name); + auto& list = topObj["functions"]; + list = json::array(); + for (auto & [fun, count] : functionCalls) { + json obj = json::object(); + if (fun->name) + obj["name"] = (std::string_view) symbols[fun->name]; else - obj.attr("name", nullptr); - if (i.first->pos) { - obj.attr("file", (const std::string &) i.first->pos.file); - obj.attr("line", i.first->pos.line); - obj.attr("column", i.first->pos.column); + obj["name"] = nullptr; + if (auto pos = positions[fun->pos]) { + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); + obj["line"] = pos.line; + obj["column"] = pos.column; } - obj.attr("count", i.second); + obj["count"] = count; + list.push_back(obj); } } { - auto list = topObj.list("attributes"); + auto list = topObj["attributes"]; + list = json::array(); for (auto & i : attrSelects) { - auto obj = list.object(); - if (i.first) { - obj.attr("file", (const std::string &) i.first.file); - obj.attr("line", i.first.line); - obj.attr("column", i.first.column); + json obj = json::object(); + if (auto pos = positions[i.first]) { + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); + obj["line"] = pos.line; + obj["column"] = pos.column; } - obj.attr("count", i.second); + obj["count"] = i.second; + list.push_back(obj); } } } if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { - auto list = topObj.list("symbols"); - symbols.dump([&](const std::string & s) { list.elem(s); }); + // XXX: overrides earlier assignment + topObj["symbols"] = json::array(); + auto &list = topObj["symbols"]; + symbols.dump([&](const std::string & s) { list.emplace_back(s); }); + } + if (outPath == "-") { + std::cerr << topObj.dump(2) << std::endl; + } else { + fs << topObj.dump(2) << std::endl; } } } -std::string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const +std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { throw TypeError({ - .msg = hintfmt("cannot coerce %1% to a string", showType()), - .errPos = pos + .msg = hintfmt("cannot coerce %1% to a string", showType()) }); } @@ -2307,7 +2620,7 @@ { Strings res; auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathExists(p)) { + if (pathAccessible(p)) { if (s.empty()) { res.push_back(p); } else { @@ -2317,14 +2630,31 @@ }; if (!evalSettings.restrictEval && !evalSettings.pureEval) { - add(getHome() + "/.nix-defexpr/channels"); - add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); - add(settings.nixStateDir + "/profiles/per-user/root/channels"); + add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels"); + add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); + add(rootChannelsDir()); } return res; } +bool EvalSettings::isPseudoUrl(std::string_view s) +{ + if (s.compare(0, 8, "channel:") == 0) return true; + size_t pos = s.find("://"); + if (pos == std::string::npos) return false; + std::string scheme(s, 0, pos); + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; +} + +std::string EvalSettings::resolvePseudoUrl(std::string_view url) +{ + if (hasPrefix(url, "channel:")) + return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; + else + return std::string(url); +} + EvalSettings evalSettings; static GlobalConfig::Register rEvalSettings(&evalSettings); diff -Nru nix-2.8.0/src/libexpr/eval.hh nix-2.16.1+dfsg/src/libexpr/eval.hh --- nix-2.8.0/src/libexpr/eval.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/eval.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "attr-set.hh" #include "types.hh" @@ -7,60 +8,71 @@ #include "symbol-table.hh" #include "config.hh" #include "experimental-features.hh" +#include "input-accessor.hh" #include #include #include #include - namespace nix { class Store; class EvalState; class StorePath; +struct DerivedPath; enum RepairFlag : bool; -typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); - +typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v); struct PrimOp { PrimOpFun fun; size_t arity; - Symbol name; + std::string name; std::vector args; const char * doc = nullptr; }; +#if HAVE_BOEHMGC + typedef std::map, traceable_allocator > > ValMap; +#else + typedef std::map ValMap; +#endif struct Env { Env * up; - unsigned short prevWith:14; // nr of levels up to next `with' environment + /** + * Number of of levels up to next `with` environment + */ + unsigned short prevWith:14; enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; Value * values[0]; }; +void printEnvBindings(const EvalState &es, const Expr & expr, const Env & env); +void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env, int lvl = 0); -void copyContext(const Value & v, PathSet & context); +std::unique_ptr mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env); +void copyContext(const Value & v, NixStringContext & context); -/* Cache for calls to addToStore(); maps source paths to the store - paths. */ -typedef std::map SrcToStore; - -std::ostream & operator << (std::ostream & str, const Value & v); +std::string printValue(const EvalState & state, const Value & v); +std::ostream & operator << (std::ostream & os, const ValueType t); +// FIXME: maybe change this to an std::variant. typedef std::pair SearchPathElem; typedef std::list SearchPath; -/* Initialise the Boehm GC, if applicable. */ +/** + * Initialise the Boehm GC, if applicable. + */ void initGC(); @@ -68,11 +80,58 @@ std::shared_ptr makeRegexCache(); +struct DebugTrace { + std::shared_ptr pos; + const Expr & expr; + const Env & env; + hintformat hint; + bool isError; +}; + +void debugError(Error * e, Env & env, Expr & expr); + +class ErrorBuilder +{ + private: + EvalState & state; + ErrorInfo info; + + ErrorBuilder(EvalState & s, ErrorInfo && i): state(s), info(i) { } + + public: + template + [[nodiscard, gnu::noinline]] + static ErrorBuilder * create(EvalState & s, const Args & ... args) + { + return new ErrorBuilder(s, ErrorInfo { .msg = hintfmt(args...) }); + } + + [[nodiscard, gnu::noinline]] + ErrorBuilder & atPos(PosIdx pos); + + [[nodiscard, gnu::noinline]] + ErrorBuilder & withTrace(PosIdx pos, const std::string_view text); + + [[nodiscard, gnu::noinline]] + ErrorBuilder & withFrameTrace(PosIdx pos, const std::string_view text); + + [[nodiscard, gnu::noinline]] + ErrorBuilder & withSuggestions(Suggestions & s); + + [[nodiscard, gnu::noinline]] + ErrorBuilder & withFrame(const Env & e, const Expr & ex); + + template + [[gnu::noinline, gnu::noreturn]] + void debugThrow(); +}; + -class EvalState +class EvalState : public std::enable_shared_from_this { public: SymbolTable symbols; + PosTable positions; const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, @@ -82,44 +141,116 @@ sOutputHash, sOutputHashAlgo, sOutputHashMode, sRecurseForDerivations, sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath, - sPrefix; - Symbol sDerivationNix; + sPrefix, + sOutputSpecified; - /* If set, force copying files to the Nix store even if they - already exist there. */ + /** + * If set, force copying files to the Nix store even if they + * already exist there. + */ RepairFlag repair; - /* The allowed filesystem paths in restricted or pure evaluation - mode. */ + /** + * The allowed filesystem paths in restricted or pure evaluation + * mode. + */ std::optional allowedPaths; Bindings emptyBindings; - /* Store used to materialise .drv files. */ + const SourcePath derivationInternal; + + /** + * Store used to materialise .drv files. + */ const ref store; - /* Store used to build stuff. */ + /** + * Store used to build stuff. + */ const ref buildStore; RootValue vCallFlake = nullptr; RootValue vImportedDrvToDerivation = nullptr; + /** + * Debugger + */ + void (* debugRepl)(ref es, const ValMap & extraEnv); + bool debugStop; + bool debugQuit; + int trylevel; + std::list debugTraces; + std::map> exprEnvs; + const std::shared_ptr getStaticEnv(const Expr & expr) const + { + auto i = exprEnvs.find(&expr); + if (i != exprEnvs.end()) + return i->second; + else + return std::shared_ptr();; + } + + void runDebugRepl(const Error * error, const Env & env, const Expr & expr); + + template + [[gnu::noinline, gnu::noreturn]] + void debugThrowLastTrace(E && error) + { + debugThrow(error, nullptr, nullptr); + } + + template + [[gnu::noinline, gnu::noreturn]] + void debugThrow(E && error, const Env * env, const Expr * expr) + { + if (debugRepl && ((env && expr) || !debugTraces.empty())) { + if (!env || !expr) { + const DebugTrace & last = debugTraces.front(); + env = &last.env; + expr = &last.expr; + } + runDebugRepl(&error, *env, *expr); + } + + throw std::move(error); + } + + // This is dangerous, but gets in line with the idea that error creation and + // throwing should not allocate on the stack of hot functions. + // as long as errors are immediately thrown, it works. + ErrorBuilder * errorBuilder; + + template + [[nodiscard, gnu::noinline]] + ErrorBuilder & error(const Args & ... args) { + errorBuilder = ErrorBuilder::create(*this, args...); + return *errorBuilder; + } + private: - SrcToStore srcToStore; - /* A cache from path names to parse trees. */ + /* Cache for calls to addToStore(); maps source paths to the store + paths. */ + std::map srcToStore; + + /** + * A cache from path names to parse trees. + */ #if HAVE_BOEHMGC - typedef std::map, traceable_allocator > > FileParseCache; + typedef std::map, traceable_allocator>> FileParseCache; #else - typedef std::map FileParseCache; + typedef std::map FileParseCache; #endif FileParseCache fileParseCache; - /* A cache from path names to values. */ + /** + * A cache from path names to values. + */ #if HAVE_BOEHMGC - typedef std::map, traceable_allocator > > FileEvalCache; + typedef std::map, traceable_allocator>> FileEvalCache; #else - typedef std::map FileEvalCache; + typedef std::map FileEvalCache; #endif FileEvalCache fileEvalCache; @@ -127,17 +258,25 @@ std::map> searchPathResolved; - /* Cache used by checkSourcePath(). */ - std::unordered_map resolvedPaths; + /** + * Cache used by checkSourcePath(). + */ + std::unordered_map resolvedPaths; - /* Cache used by prim_match(). */ + /** + * Cache used by prim_match(). + */ std::shared_ptr regexCache; #if HAVE_BOEHMGC - /* Allocation cache for GC'd Value objects. */ + /** + * Allocation cache for GC'd Value objects. + */ std::shared_ptr valueAllocCache; - /* Allocation cache for size-1 Env objects. */ + /** + * Allocation cache for size-1 Env objects. + */ std::shared_ptr env1AllocCache; #endif @@ -153,135 +292,222 @@ SearchPath getSearchPath() { return searchPath; } - /* Allow access to a path. */ + /** + * Return a `SourcePath` that refers to `path` in the root + * filesystem. + */ + SourcePath rootPath(CanonPath path); + + /** + * Allow access to a path. + */ void allowPath(const Path & path); - /* Allow access to a store path. Note that this gets remapped to - the real store path if `store` is a chroot store. */ + /** + * Allow access to a store path. Note that this gets remapped to + * the real store path if `store` is a chroot store. + */ void allowPath(const StorePath & storePath); - /* Allow access to a store path and return it as a string. */ + /** + * Allow access to a store path and return it as a string. + */ void allowAndSetStorePathString(const StorePath & storePath, Value & v); - /* Check whether access to a path is allowed and throw an error if - not. Otherwise return the canonicalised path. */ - Path checkSourcePath(const Path & path); + /** + * Check whether access to a path is allowed and throw an error if + * not. Otherwise return the canonicalised path. + */ + SourcePath checkSourcePath(const SourcePath & path); void checkURI(const std::string & uri); - /* When using a diverted store and 'path' is in the Nix store, map - 'path' to the diverted location (e.g. /nix/store/foo is mapped - to /home/alice/my-nix/nix/store/foo). However, this is only - done if the context is not empty, since otherwise we're - probably trying to read from the actual /nix/store. This is - intended to distinguish between import-from-derivation and - sources stored in the actual /nix/store. */ - Path toRealPath(const Path & path, const PathSet & context); - - /* Parse a Nix expression from the specified file. */ - Expr * parseExprFromFile(const Path & path); - Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv); - - /* Parse a Nix expression from the specified string. */ - Expr * parseExprFromString(std::string s, const Path & basePath, StaticEnv & staticEnv); - Expr * parseExprFromString(std::string s, const Path & basePath); + /** + * When using a diverted store and 'path' is in the Nix store, map + * 'path' to the diverted location (e.g. /nix/store/foo is mapped + * to /home/alice/my-nix/nix/store/foo). However, this is only + * done if the context is not empty, since otherwise we're + * probably trying to read from the actual /nix/store. This is + * intended to distinguish between import-from-derivation and + * sources stored in the actual /nix/store. + */ + Path toRealPath(const Path & path, const NixStringContext & context); + + /** + * Parse a Nix expression from the specified file. + */ + Expr * parseExprFromFile(const SourcePath & path); + Expr * parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv); + + /** + * Parse a Nix expression from the specified string. + */ + Expr * parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr & staticEnv); + Expr * parseExprFromString(std::string s, const SourcePath & basePath); Expr * parseStdin(); - /* Evaluate an expression read from the given file to normal - form. Optionally enforce that the top-level expression is - trivial (i.e. doesn't require arbitrary computation). */ - void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); + /** + * Evaluate an expression read from the given file to normal + * form. Optionally enforce that the top-level expression is + * trivial (i.e. doesn't require arbitrary computation). + */ + void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false); - /* Like `cacheFile`, but with an already parsed expression. */ + /** + * Like `evalFile`, but with an already parsed expression. + */ void cacheFile( - const Path & path, - const Path & resolvedPath, + const SourcePath & path, + const SourcePath & resolvedPath, Expr * e, Value & v, bool mustBeTrivial = false); void resetFileCache(); - /* Look up a file in the search path. */ - Path findFile(const std::string_view path); - Path findFile(SearchPath & searchPath, const std::string_view path, const Pos & pos = noPos); + /** + * Look up a file in the search path. + */ + SourcePath findFile(const std::string_view path); + SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); - /* If the specified search path element is a URI, download it. */ + /** + * If the specified search path element is a URI, download it. + */ std::pair resolveSearchPathElem(const SearchPathElem & elem); - /* Evaluate an expression to normal form, storing the result in - value `v'. */ + /** + * Evaluate an expression to normal form + * + * @param [out] v The resulting is stored here. + */ void eval(Expr * e, Value & v); - /* Evaluation the expression, then verify that it has the expected - type. */ + /** + * Evaluation the expression, then verify that it has the expected + * type. + */ inline bool evalBool(Env & env, Expr * e); - inline bool evalBool(Env & env, Expr * e, const Pos & pos); - inline void evalAttrs(Env & env, Expr * e, Value & v); + inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx); + inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx); - /* If `v' is a thunk, enter it and overwrite `v' with the result - of the evaluation of the thunk. If `v' is a delayed function - application, call the function and overwrite `v' with the - result. Otherwise, this is a no-op. */ - inline void forceValue(Value & v, const Pos & pos); + /** + * If `v` is a thunk, enter it and overwrite `v` with the result + * of the evaluation of the thunk. If `v` is a delayed function + * application, call the function and overwrite `v` with the + * result. Otherwise, this is a no-op. + */ + inline void forceValue(Value & v, const PosIdx pos); template inline void forceValue(Value & v, Callable getPos); - /* Force a value, then recursively force list elements and - attributes. */ + /** + * Force a value, then recursively force list elements and + * attributes. + */ void forceValueDeep(Value & v); - /* Force `v', and then verify that it has the expected type. */ - NixInt forceInt(Value & v, const Pos & pos); - NixFloat forceFloat(Value & v, const Pos & pos); - bool forceBool(Value & v, const Pos & pos); + /** + * Force `v`, and then verify that it has the expected type. + */ + NixInt forceInt(Value & v, const PosIdx pos, std::string_view errorCtx); + NixFloat forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx); + bool forceBool(Value & v, const PosIdx pos, std::string_view errorCtx); - void forceAttrs(Value & v, const Pos & pos); + void forceAttrs(Value & v, const PosIdx pos, std::string_view errorCtx); template - inline void forceAttrs(Value & v, Callable getPos); + inline void forceAttrs(Value & v, Callable getPos, std::string_view errorCtx); - inline void forceList(Value & v, const Pos & pos); - void forceFunction(Value & v, const Pos & pos); // either lambda or primop - std::string_view forceString(Value & v, const Pos & pos = noPos); - std::string_view forceString(Value & v, PathSet & context, const Pos & pos = noPos); - std::string_view forceStringNoCtx(Value & v, const Pos & pos = noPos); + inline void forceList(Value & v, const PosIdx pos, std::string_view errorCtx); + /** + * @param v either lambda or primop + */ + void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); + std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx); + std::string_view forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx); + std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx); + + [[gnu::noinline]] + void addErrorTrace(Error & e, const char * s, const std::string & s2) const; + [[gnu::noinline]] + void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const; - /* Return true iff the value `v' denotes a derivation (i.e. a - set with attribute `type = "derivation"'). */ +public: + /** + * @return true iff the value `v` denotes a derivation (i.e. a + * set with attribute `type = "derivation"`). + */ bool isDerivation(Value & v); - std::optional tryAttrsToString(const Pos & pos, Value & v, - PathSet & context, bool coerceMore = false, bool copyToStore = true); + std::optional tryAttrsToString(const PosIdx pos, Value & v, + NixStringContext & context, bool coerceMore = false, bool copyToStore = true); - /* String coercion. Converts strings, paths and derivations to a - string. If `coerceMore' is set, also converts nulls, integers, - booleans and lists to a string. If `copyToStore' is set, - referenced paths are copied to the Nix store as a side effect. */ - BackedStringView coerceToString(const Pos & pos, Value & v, PathSet & context, + /** + * String coercion. + * + * Converts strings, paths and derivations to a + * string. If `coerceMore` is set, also converts nulls, integers, + * booleans and lists to a string. If `copyToStore` is set, + * referenced paths are copied to the Nix store as a side effect. + */ + BackedStringView coerceToString(const PosIdx pos, Value & v, NixStringContext & context, + std::string_view errorCtx, bool coerceMore = false, bool copyToStore = true, bool canonicalizePath = true); - std::string copyPathToStore(PathSet & context, const Path & path); + StorePath copyPathToStore(NixStringContext & context, const SourcePath & path); + + /** + * Path coercion. + * + * Converts strings, paths and derivations to a + * path. The result is guaranteed to be a canonicalised, absolute + * path. Nothing is copied to the store. + */ + SourcePath coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); + + /** + * Like coerceToPath, but the result must be a store path. + */ + StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); - /* Path coercion. Converts strings, paths and derivations to a - path. The result is guaranteed to be a canonicalised, absolute - path. Nothing is copied to the store. */ - Path coerceToPath(const Pos & pos, Value & v, PathSet & context); + /** + * Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only. + */ + std::pair coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); - /* Like coerceToPath, but the result must be a store path. */ - StorePath coerceToStorePath(const Pos & pos, Value & v, PathSet & context); + /** + * Coerce to `DerivedPath`. + * + * Must be a string which is either a literal store path or a + * "placeholder (see `DownstreamPlaceholder`). + * + * Even more importantly, the string context must be exactly one + * element, which is either a `NixStringContextElem::Opaque` or + * `NixStringContextElem::Built`. (`NixStringContextEleme::DrvDeep` + * is not permitted). + * + * The string is parsed based on the context --- the context is the + * source of truth, and ultimately tells us what we want, and then + * we ensure the string corresponds to it. + */ + DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); public: - /* The base environment, containing the builtin functions and - values. */ + /** + * The base environment, containing the builtin functions and + * values. + */ Env & baseEnv; - /* The same, but used during parsing to resolve variables. */ - StaticEnv staticBaseEnv; // !!! should be private + /** + * The same, but used during parsing to resolve variables. + */ + std::shared_ptr staticBaseEnv; // !!! should be private private: @@ -305,7 +531,7 @@ struct Doc { Pos pos; - std::optional name; + std::optional name; size_t arity; std::vector args; const char * doc; @@ -321,35 +547,45 @@ friend struct ExprAttrs; friend struct ExprLet; - Expr * parse(char * text, size_t length, FileOrigin origin, const PathView path, - const PathView basePath, StaticEnv & staticEnv); + Expr * parse( + char * text, + size_t length, + Pos::Origin origin, + const SourcePath & basePath, + std::shared_ptr & staticEnv); public: - /* Do a deep equality test between two values. That is, list - elements and attributes are compared recursively. */ - bool eqValues(Value & v1, Value & v2); + /** + * Do a deep equality test between two values. That is, list + * elements and attributes are compared recursively. + */ + bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); bool isFunctor(Value & fun); // FIXME: use std::span - void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos); + void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos); - void callFunction(Value & fun, Value & arg, Value & vRes, const Pos & pos) + void callFunction(Value & fun, Value & arg, Value & vRes, const PosIdx pos) { Value * args[] = {&arg}; callFunction(fun, 1, args, vRes, pos); } - /* Automatically call a function for which each argument has a - default value or has a binding in the `args' map. */ + /** + * Automatically call a function for which each argument has a + * default value or has a binding in the `args` map. + */ void autoCallFunction(Bindings & args, Value & fun, Value & res); - /* Allocation primitives. */ + /** + * Allocation primitives. + */ inline Value * allocValue(); inline Env & allocEnv(size_t size); - Value * allocAttr(Value & vAttrs, const Symbol & name); + Value * allocAttr(Value & vAttrs, Symbol name); Value * allocAttr(Value & vAttrs, std::string_view name); Bindings * allocBindings(size_t capacity); @@ -361,17 +597,51 @@ void mkList(Value & v, size_t length); void mkThunk_(Value & v, Expr * expr); - void mkPos(Value & v, ptr pos); + void mkPos(Value & v, PosIdx pos); + + /** + * Create a string representing a store path. + * + * The string is the printed store path with a context containing a single + * `NixStringContextElem::Opaque` element of that store path. + */ + void mkStorePathString(const StorePath & storePath, Value & v); + + /** + * Create a string representing a `DerivedPath::Built`. + * + * The string is the printed store path with a context containing a single + * `NixStringContextElem::Built` element of the drv path and output name. + * + * @param value Value we are settings + * + * @param drvPath Path the drv whose output we are making a string for + * + * @param outputName Name of the output + * + * @param optOutputPath Optional output path for that string. Must + * be passed if and only if output store object is input-addressed. + * Will be printed to form string if passed, otherwise a placeholder + * will be used (see `DownstreamPlaceholder`). + */ + void mkOutputString( + Value & value, + const StorePath & drvPath, + const std::string outputName, + std::optional optOutputPath); - void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos); + void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); - /* Print statistics. */ + /** + * Print statistics. + */ void printStats(); - /* Realise the given context, and return a mapping from the placeholders + /** + * Realise the given context, and return a mapping from the placeholders * used to construct the associated value to their final store path */ - [[nodiscard]] StringMap realiseContext(const PathSet & context); + [[nodiscard]] StringMap realiseContext(const NixStringContext & context); private: @@ -391,7 +661,7 @@ bool countCalls; - typedef std::map PrimOpCalls; + typedef std::map PrimOpCalls; PrimOpCalls primOpCalls; typedef std::map FunctionCalls; @@ -399,7 +669,7 @@ void incrFunctionCall(ExprLambda * fun); - typedef std::map AttrSelects; + typedef std::map AttrSelects; AttrSelects attrSelects; friend struct ExprOpUpdate; @@ -410,24 +680,34 @@ friend struct ExprFloat; friend struct ExprPath; friend struct ExprSelect; - friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); - friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v); - friend void prim_split(EvalState & state, const Pos & pos, Value * * args, Value & v); + friend void prim_getAttr(EvalState & state, const PosIdx pos, Value * * args, Value & v); + friend void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v); + friend void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v); friend struct Value; }; +struct DebugTraceStacker { + DebugTraceStacker(EvalState & evalState, DebugTrace t); + ~DebugTraceStacker() + { + // assert(evalState.debugTraces.front() == trace); + evalState.debugTraces.pop_front(); + } + EvalState & evalState; + DebugTrace trace; +}; -/* Return a string representing the type of the value `v'. */ +/** + * @return A string representing the type of the value `v`. + */ std::string_view showType(ValueType type); std::string showType(const Value & v); -/* Decode a context string ‘!!’ into a pair . */ -NixStringContextElem decodeContext(const Store & store, std::string_view s); - -/* If `path' refers to a directory, then append "/default.nix". */ -Path resolveExprPath(Path path); +/** + * If `path` refers to a directory, then append "/default.nix". + */ +SourcePath resolveExprPath(const SourcePath & path); struct InvalidPathError : EvalError { @@ -444,6 +724,10 @@ static Strings getDefaultNixPath(); + static bool isPseudoUrl(std::string_view s); + + static std::string resolvePseudoUrl(std::string_view url); + Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", "Whether builtin functions that allow executing native code should be enabled."}; @@ -461,7 +745,13 @@ )"}; Setting pureEval{this, false, "pure-eval", - "Whether to restrict file system and network access to files specified by cryptographic hash."}; + R"( + Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: + + - Restrict file system and network access to files specified by cryptographic hash + - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + )" + }; Setting enableImportFromDerivation{ this, true, "allow-import-from-derivation", @@ -501,12 +791,28 @@ Setting useEvalCache{this, true, "eval-cache", "Whether to use the flake evaluation cache."}; + + Setting ignoreExceptionsDuringTry{this, false, "ignore-try", + R"( + If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in + debug mode (using the --debugger flag). By default the debugger will pause on all exceptions. + )"}; + + Setting traceVerbose{this, false, "trace-verbose", + "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; }; extern EvalSettings evalSettings; static const std::string corepkgsPrefix{"/__corepkgs__/"}; +template +void ErrorBuilder::debugThrow() +{ + // NOTE: We always use the -LastTrace version as we push the new trace in withFrame() + state.debugThrowLastTrace(ErrorType(info)); +} + } #include "eval-inline.hh" diff -Nru nix-2.8.0/src/libexpr/eval-inline.hh nix-2.16.1+dfsg/src/libexpr/eval-inline.hh --- nix-2.8.0/src/libexpr/eval-inline.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/eval-inline.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,30 +1,13 @@ #pragma once +///@file #include "eval.hh" -#define LocalNoInline(f) static f __attribute__((noinline)); f -#define LocalNoInlineNoReturn(f) static f __attribute__((noinline, noreturn)); f - namespace nix { -LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s)) -{ - throw EvalError({ - .msg = hintfmt(s), - .errPos = pos - }); -} - -LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v)) -{ - throw TypeError({ - .msg = hintfmt(s, showType(v)), - .errPos = pos - }); -} - - -/* Note: Various places expect the allocated memory to be zeroed. */ +/** + * Note: Various places expect the allocated memory to be zeroed. + */ [[gnu::always_inline]] inline void * allocBytes(size_t n) { @@ -99,7 +82,7 @@ [[gnu::always_inline]] -void EvalState::forceValue(Value & v, const Pos & pos) +void EvalState::forceValue(Value & v, const PosIdx pos) { forceValue(v, [&]() { return pos; }); } @@ -123,33 +106,36 @@ else if (v.isApp()) callFunction(*v.app.left, *v.app.right, v, noPos); else if (v.isBlackhole()) - throwEvalError(getPos(), "infinite recursion encountered"); + error("infinite recursion encountered").atPos(getPos()).template debugThrow(); } [[gnu::always_inline]] -inline void EvalState::forceAttrs(Value & v, const Pos & pos) +inline void EvalState::forceAttrs(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceAttrs(v, [&]() { return pos; }); + forceAttrs(v, [&]() { return pos; }, errorCtx); } template [[gnu::always_inline]] -inline void EvalState::forceAttrs(Value & v, Callable getPos) +inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view errorCtx) { - forceValue(v, getPos); - if (v.type() != nAttrs) - throwTypeError(getPos(), "value is %1% while a set was expected", v); + forceValue(v, noPos); + if (v.type() != nAttrs) { + PosIdx pos = getPos(); + error("value is %1% while a set was expected", showType(v)).withTrace(pos, errorCtx).debugThrow(); + } } [[gnu::always_inline]] -inline void EvalState::forceList(Value & v, const Pos & pos) +inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (!v.isList()) - throwTypeError(pos, "value is %1% while a list was expected", v); + forceValue(v, noPos); + if (!v.isList()) { + error("value is %1% while a list was expected", showType(v)).withTrace(pos, errorCtx).debugThrow(); + } } diff -Nru nix-2.8.0/src/libexpr/fetchurl.nix nix-2.16.1+dfsg/src/libexpr/fetchurl.nix --- nix-2.8.0/src/libexpr/fetchurl.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/fetchurl.nix 2023-06-06 10:55:23.000000000 +0000 @@ -12,13 +12,13 @@ , executable ? false , unpack ? false , name ? baseNameOf (toString url) +, impure ? false }: -derivation { +derivation ({ builder = "builtin:fetchurl"; # New-style output content requirements. - inherit outputHashAlgo outputHash; outputHashMode = if unpack || executable then "recursive" else "flat"; inherit name url executable unpack; @@ -38,4 +38,6 @@ # To make "nix-prefetch-url" work. urls = [ url ]; -} +} // (if impure + then { __impure = true; } + else { inherit outputHashAlgo outputHash; })) diff -Nru nix-2.8.0/src/libexpr/flake/call-flake.nix nix-2.16.1+dfsg/src/libexpr/flake/call-flake.nix --- nix-2.8.0/src/libexpr/flake/call-flake.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/flake/call-flake.nix 2023-06-06 10:55:23.000000000 +0000 @@ -16,7 +16,9 @@ subdir = if key == lockFile.root then rootSubdir else node.locked.dir or ""; - flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix"); + outPath = sourceInfo + ((if subdir == "" then "" else "/") + subdir); + + flake = import (outPath + "/flake.nix"); inputs = builtins.mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}) @@ -43,7 +45,21 @@ outputs = flake.outputs (inputs // { self = result; }); - result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; }; + result = + outputs + # We add the sourceInfo attribute for its metadata, as they are + # relevant metadata for the flake. However, the outPath of the + # sourceInfo does not necessarily match the outPath of the flake, + # as the flake may be in a subdirectory of a source. + # This is shadowed in the next // + // sourceInfo + // { + # This shadows the sourceInfo.outPath + inherit outPath; + + inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake"; + }; + in if node.flake or true then assert builtins.isFunction flake.outputs; diff -Nru nix-2.8.0/src/libexpr/flake/config.cc nix-2.16.1+dfsg/src/libexpr/flake/config.cc --- nix-2.8.0/src/libexpr/flake/config.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/flake/config.cc 2023-06-06 10:55:23.000000000 +0000 @@ -31,7 +31,7 @@ void ConfigFile::apply() { - std::set whitelist{"bash-prompt", "bash-prompt-suffix", "flake-registry"}; + std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lockfile-summary"}; for (auto & [name, value] : settings) { @@ -50,15 +50,13 @@ else assert(false); - if (!whitelist.count(baseName)) { - auto trustedList = readTrustedList(); - + if (!whitelist.count(baseName) && !nix::fetchSettings.acceptFlakeConfig) { bool trusted = false; - if (nix::fetchSettings.acceptFlakeConfig){ - trusted = true; - } else if (auto saved = get(get(trustedList, name).value_or(std::map()), valueS)) { + auto trustedList = readTrustedList(); + auto tlname = get(trustedList, name); + if (auto saved = tlname ? get(*tlname, valueS) : nullptr) { trusted = *saved; - warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS); + printInfo("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name, valueS); } else { // FIXME: filter ANSI escapes, newlines, \r, etc. if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') { @@ -69,9 +67,8 @@ writeTrustedList(trustedList); } } - if (!trusted) { - warn("ignoring untrusted flake configuration setting '%s'", name); + warn("ignoring untrusted flake configuration setting '%s'.\nPass '%s' to trust it", name, "--accept-flake-config"); continue; } } diff -Nru nix-2.8.0/src/libexpr/flake/flake.cc nix-2.16.1+dfsg/src/libexpr/flake/flake.cc --- nix-2.8.0/src/libexpr/flake/flake.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/flake/flake.cc 2023-06-06 10:55:23.000000000 +0000 @@ -72,7 +72,7 @@ return {std::move(tree), resolvedRef, lockedRef}; } -static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos) +static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) { if (value.isThunk() && value.isTrivial()) state.forceValue(value, pos); @@ -80,20 +80,20 @@ static void expectType(EvalState & state, ValueType type, - Value & value, const Pos & pos) + Value & value, const PosIdx pos) { forceTrivialValue(state, value, pos); if (value.type() != type) throw Error("expected %s but got %s at %s", - showType(type), showType(value.type()), pos); + showType(type), showType(value.type()), state.positions[pos]); } static std::map parseFlakeInputs( - EvalState & state, Value * value, const Pos & pos, + EvalState & state, Value * value, const PosIdx pos, const std::optional & baseDir, InputPath lockRootPath); static FlakeInput parseFlakeInput(EvalState & state, - const std::string & inputName, Value * value, const Pos & pos, + const std::string & inputName, Value * value, const PosIdx pos, const std::optional & baseDir, InputPath lockRootPath) { expectType(state, nAttrs, *value, pos); @@ -111,37 +111,43 @@ for (nix::Attr attr : *(value->attrs)) { try { if (attr.name == sUrl) { - expectType(state, nString, *attr.value, *attr.pos); + expectType(state, nString, *attr.value, attr.pos); url = attr.value->string.s; attrs.emplace("url", *url); } else if (attr.name == sFlake) { - expectType(state, nBool, *attr.value, *attr.pos); + expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean; } else if (attr.name == sInputs) { - input.overrides = parseFlakeInputs(state, attr.value, *attr.pos, baseDir, lockRootPath); + input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath); } else if (attr.name == sFollows) { - expectType(state, nString, *attr.value, *attr.pos); + expectType(state, nString, *attr.value, attr.pos); auto follows(parseInputPath(attr.value->string.s)); follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end()); input.follows = follows; } else { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (attr.value->type()) { case nString: - attrs.emplace(attr.name, attr.value->string.s); + attrs.emplace(state.symbols[attr.name], attr.value->string.s); break; case nBool: - attrs.emplace(attr.name, Explicit { attr.value->boolean }); + attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean }); break; case nInt: - attrs.emplace(attr.name, (long unsigned int)attr.value->integer); + attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer); break; default: throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", - attr.name, showType(*attr.value)); + state.symbols[attr.name], showType(*attr.value)); } + #pragma GCC diagnostic pop } } catch (Error & e) { - e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name)); + e.addTrace( + state.positions[attr.pos], + hintfmt("while evaluating flake attribute '%s'", state.symbols[attr.name])); throw; } } @@ -150,13 +156,13 @@ try { input.ref = FlakeRef::fromAttrs(attrs); } catch (Error & e) { - e.addTrace(pos, hintfmt("in flake input")); + e.addTrace(state.positions[pos], hintfmt("while evaluating flake input")); throw; } else { attrs.erase("url"); if (!attrs.empty()) - throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos); + throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, state.positions[pos]); if (url) input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake); } @@ -168,7 +174,7 @@ } static std::map parseFlakeInputs( - EvalState & state, Value * value, const Pos & pos, + EvalState & state, Value * value, const PosIdx pos, const std::optional & baseDir, InputPath lockRootPath) { std::map inputs; @@ -176,11 +182,11 @@ expectType(state, nAttrs, *value, pos); for (nix::Attr & inputAttr : *(*value).attrs) { - inputs.emplace(inputAttr.name, + inputs.emplace(state.symbols[inputAttr.name], parseFlakeInput(state, - inputAttr.name, + state.symbols[inputAttr.name], inputAttr.value, - *inputAttr.pos, + inputAttr.pos, baseDir, lockRootPath)); } @@ -216,30 +222,30 @@ throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); Value vInfo; - state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack + state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack - expectType(state, nAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0)); + expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1)); if (auto description = vInfo.attrs->get(state.sDescription)) { - expectType(state, nString, *description->value, *description->pos); + expectType(state, nString, *description->value, description->pos); flake.description = description->value->string.s; } auto sInputs = state.symbols.create("inputs"); if (auto inputs = vInfo.attrs->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos, flakeDir, lockRootPath); + flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakeDir, lockRootPath); auto sOutputs = state.symbols.create("outputs"); if (auto outputs = vInfo.attrs->get(sOutputs)) { - expectType(state, nFunction, *outputs->value, *outputs->pos); + expectType(state, nFunction, *outputs->value, outputs->pos); if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) { for (auto & formal : outputs->value->lambda.fun->formals->formals) { if (formal.name != state.sSelf) - flake.inputs.emplace(formal.name, FlakeInput { - .ref = parseFlakeRef(formal.name) + flake.inputs.emplace(state.symbols[formal.name], FlakeInput { + .ref = parseFlakeRef(state.symbols[formal.name]) }); } } @@ -250,35 +256,41 @@ auto sNixConfig = state.symbols.create("nixConfig"); if (auto nixConfig = vInfo.attrs->get(sNixConfig)) { - expectType(state, nAttrs, *nixConfig->value, *nixConfig->pos); + expectType(state, nAttrs, *nixConfig->value, nixConfig->pos); for (auto & setting : *nixConfig->value->attrs) { - forceTrivialValue(state, *setting.value, *setting.pos); + forceTrivialValue(state, *setting.value, setting.pos); if (setting.value->type() == nString) - flake.config.settings.insert({setting.name, std::string(state.forceStringNoCtx(*setting.value, *setting.pos))}); + flake.config.settings.emplace( + state.symbols[setting.name], + std::string(state.forceStringNoCtx(*setting.value, setting.pos, ""))); else if (setting.value->type() == nPath) { - PathSet emptyContext = {}; + NixStringContext emptyContext = {}; flake.config.settings.emplace( - setting.name, - state.coerceToString(*setting.pos, *setting.value, emptyContext, false, true, true) .toOwned()); + state.symbols[setting.name], + state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true) .toOwned()); } else if (setting.value->type() == nInt) - flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)}); + flake.config.settings.emplace( + state.symbols[setting.name], + state.forceInt(*setting.value, setting.pos, "")); else if (setting.value->type() == nBool) - flake.config.settings.insert({setting.name, Explicit { state.forceBool(*setting.value, *setting.pos) }}); + flake.config.settings.emplace( + state.symbols[setting.name], + Explicit { state.forceBool(*setting.value, setting.pos, "") }); else if (setting.value->type() == nList) { std::vector ss; for (auto elem : setting.value->listItems()) { if (elem->type() != nString) throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", - setting.name, showType(*setting.value)); - ss.emplace_back(state.forceStringNoCtx(*elem, *setting.pos)); + state.symbols[setting.name], showType(*setting.value)); + ss.emplace_back(state.forceStringNoCtx(*elem, setting.pos, "")); } - flake.config.settings.insert({setting.name, ss}); + flake.config.settings.emplace(state.symbols[setting.name], ss); } else throw TypeError("flake configuration setting '%s' is %s", - setting.name, showType(*setting.value)); + state.symbols[setting.name], showType(*setting.value)); } } @@ -288,7 +300,7 @@ attr.name != sOutputs && attr.name != sNixConfig) throw Error("flake '%s' has an unsupported attribute '%s', at %s", - lockedRef, attr.name, *attr.pos); + lockedRef, state.symbols[attr.name], state.positions[attr.pos]); } return flake; @@ -312,7 +324,7 @@ const FlakeRef & topRef, const LockFlags & lockFlags) { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); FlakeCache flakeCache; @@ -326,14 +338,17 @@ } try { + if (!fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { + throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false"); + } // FIXME: symlink attack auto oldLockFile = LockFile::read( - flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); + lockFlags.referenceLockFilePath.value_or( + flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock")); debug("old lock file: %s", oldLockFile); - // FIXME: check whether all overrides are used. std::map overrides; std::set overridesUsed, updatesUsed; @@ -346,7 +361,7 @@ std::function node, + ref node, const InputPath & inputPathPrefix, std::shared_ptr oldNode, const InputPath & lockRootPath, @@ -355,9 +370,15 @@ computeLocks; computeLocks = [&]( + /* The inputs of this node, either from flake.nix or + flake.lock. */ const FlakeInputs & flakeInputs, - std::shared_ptr node, + /* The node whose locks are to be updated.*/ + ref node, + /* The path to this node in the lock file graph. */ const InputPath & inputPathPrefix, + /* The old node, if any, from which locks can be + copied. */ std::shared_ptr oldNode, const InputPath & lockRootPath, const Path & parentPath, @@ -376,6 +397,18 @@ } } + /* Check whether this input has overrides for a + non-existent input. */ + for (auto [inputPath, inputOverride] : overrides) { + auto inputPath2(inputPath); + auto follow = inputPath2.back(); + inputPath2.pop_back(); + if (inputPath2 == inputPathPrefix && !flakeInputs.count(follow)) + warn( + "input '%s' has an override for a non-existent input '%s'", + printInputPath(inputPathPrefix), follow); + } + /* Go over the flake inputs, resolve/fetch them if necessary (i.e. if they're new or the flakeref changed from what's in the lock file). */ @@ -433,7 +466,7 @@ /* Copy the input from the old lock since its flakeref didn't change and there is no override from a higher level flake. */ - auto childNode = std::make_shared( + auto childNode = make_ref( oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); node->inputs.insert_or_assign(id, childNode); @@ -462,14 +495,14 @@ .isFlake = (*lockedNode)->isFlake, }); } else if (auto follows = std::get_if<1>(&i.second)) { - if (! trustLock) { + if (!trustLock) { // It is possible that the flake has changed, - // so we must confirm all the follows that are in the lockfile are also in the flake. + // so we must confirm all the follows that are in the lock file are also in the flake. auto overridePath(inputPath); overridePath.push_back(i.first); auto o = overrides.find(overridePath); // If the override disappeared, we have to refetch the flake, - // since some of the inputs may not be present in the lockfile. + // since some of the inputs may not be present in the lock file. if (o == overrides.end()) { mustRefetch = true; // There's no point populating the rest of the fake inputs, @@ -502,8 +535,17 @@ this input. */ debug("creating new input '%s'", inputPathS); - if (!lockFlags.allowMutable && !input.ref->input.isLocked()) - throw Error("cannot update flake input '%s' in pure mode", inputPathS); + if (!lockFlags.allowUnlocked && !input.ref->input.isLocked()) + throw Error("cannot update unlocked flake input '%s' in pure mode", inputPathS); + + /* Note: in case of an --override-input, we use + the *original* ref (input2.ref) for the + "original" field, rather than the + override. This ensures that the override isn't + nuked the next time we update the lock + file. That is, overrides are sticky unless you + use --no-write-lock-file. */ + auto ref = input2.ref ? *input2.ref : *input.ref; if (input.isFlake) { Path localPath = parentPath; @@ -516,15 +558,7 @@ auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache, inputPath); - /* Note: in case of an --override-input, we use - the *original* ref (input2.ref) for the - "original" field, rather than the - override. This ensures that the override isn't - nuked the next time we update the lock - file. That is, overrides are sticky unless you - use --no-write-lock-file. */ - auto childNode = std::make_shared( - inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref); + auto childNode = make_ref(inputFlake.lockedRef, ref); node->inputs.insert_or_assign(id, childNode); @@ -544,15 +578,19 @@ oldLock ? std::dynamic_pointer_cast(oldLock) : LockFile::read( - inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root, - oldLock ? lockRootPath : inputPath, localPath, false); + inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), + oldLock ? lockRootPath : inputPath, + localPath, + false); } else { auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, *input.ref, useRegistries, flakeCache); - node->inputs.insert_or_assign(id, - std::make_shared(lockedRef, *input.ref, false)); + + auto childNode = make_ref(lockedRef, ref, false); + + node->inputs.insert_or_assign(id, childNode); } } @@ -567,8 +605,13 @@ auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); computeLocks( - flake.inputs, newLockFile.root, {}, - lockFlags.recreateLockFile ? nullptr : oldLockFile.root, {}, parentPath, false); + flake.inputs, + newLockFile.root, + {}, + lockFlags.recreateLockFile ? nullptr : oldLockFile.root.get_ptr(), + {}, + parentPath, + false); for (auto & i : lockFlags.inputOverrides) if (!overridesUsed.count(i.first)) @@ -584,39 +627,45 @@ debug("new lock file: %s", newLockFile); + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto sourcePath = topRef.input.getSourcePath(); + auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; + if (lockFlags.outputLockFilePath) { + outputLockFilePath = lockFlags.outputLockFilePath; + } + /* Check whether we need to / can write the new lock file. */ - if (!(newLockFile == oldLockFile)) { + if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { auto diff = LockFile::diff(oldLockFile, newLockFile); if (lockFlags.writeLockFile) { - if (auto sourcePath = topRef.input.getSourcePath()) { - if (!newLockFile.isImmutable()) { + if (outputLockFilePath) { + if (auto unlockedInput = newLockFile.isUnlocked()) { if (fetchSettings.warnDirty) - warn("will not write lock file of flake '%s' because it has a mutable input", topRef); + warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; - - auto path = *sourcePath + "/" + relPath; - - bool lockFileExists = pathExists(path); + bool lockFileExists = pathExists(*outputLockFilePath); if (lockFileExists) { auto s = chomp(diff); if (s.empty()) - warn("updating lock file '%s'", path); + warn("updating lock file '%s'", *outputLockFilePath); else - warn("updating lock file '%s':\n%s", path, s); + warn("updating lock file '%s':\n%s", *outputLockFilePath, s); } else - warn("creating lock file '%s'", path); + warn("creating lock file '%s'", *outputLockFilePath); - newLockFile.write(path); + newLockFile.write(*outputLockFilePath); std::optional commitMessage = std::nullopt; if (lockFlags.commitLockFile) { + if (lockFlags.outputLockFilePath) { + throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); + } std::string cm; cm = fetchSettings.commitLockFileSummary.get(); @@ -696,7 +745,7 @@ state.vCallFlake = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "call-flake.nix.gen.hh" - , "/"), **state.vCallFlake); + , CanonPath::root), **state.vCallFlake); } state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); @@ -704,19 +753,20 @@ state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); } -static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - std::string flakeRefS(state.forceStringNoCtx(*args[0], pos)); + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); auto flakeRef = parseFlakeRef(flakeRefS, {}, true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) - throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, pos); + throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); callFlake(state, lockFlake(state, flakeRef, LockFlags { .updateLockFile = false, + .writeLockFile = false, .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries, - .allowMutable = !evalSettings.pureEval, + .allowUnlocked = !evalSettings.pureEval, }), v); } diff -Nru nix-2.8.0/src/libexpr/flake/flake.hh nix-2.16.1+dfsg/src/libexpr/flake/flake.hh --- nix-2.8.0/src/libexpr/flake/flake.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/flake/flake.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "flakeref.hh" @@ -17,7 +18,8 @@ typedef std::map FlakeInputs; -/* FlakeInput is the 'Flake'-level parsed form of the "input" entries +/** + * FlakeInput is the 'Flake'-level parsed form of the "input" entries * in the flake file. * * A FlakeInput is normally constructed by the 'parseFlakeInput' @@ -41,7 +43,12 @@ struct FlakeInput { std::optional ref; - bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path + /** + * true = process flake to get outputs + * + * false = (fetched) static source path + */ + bool isFlake = true; std::optional follows; FlakeInputs overrides; }; @@ -55,23 +62,42 @@ void apply(); }; -/* The contents of a flake.nix file. */ +/** + * The contents of a flake.nix file. + */ struct Flake { - FlakeRef originalRef; // the original flake specification (by the user) - FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake - FlakeRef lockedRef; // the specific local store result of invoking the fetcher - bool forceDirty = false; // pretend that 'lockedRef' is dirty + /** + * The original flake specification (by the user) + */ + FlakeRef originalRef; + /** + * registry references and caching resolved to the specific underlying flake + */ + FlakeRef resolvedRef; + /** + * the specific local store result of invoking the fetcher + */ + FlakeRef lockedRef; + /** + * pretend that 'lockedRef' is dirty + */ + bool forceDirty = false; std::optional description; std::shared_ptr sourceInfo; FlakeInputs inputs; - ConfigFile config; // 'nixConfig' attribute + /** + * 'nixConfig' attribute + */ + ConfigFile config; ~Flake(); }; Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); -/* Fingerprint of a locked flake; used as a cache key. */ +/** + * Fingerprint of a locked flake; used as a cache key. + */ typedef Hash Fingerprint; struct LockedFlake @@ -84,44 +110,72 @@ struct LockFlags { - /* Whether to ignore the existing lock file, creating a new one - from scratch. */ + /** + * Whether to ignore the existing lock file, creating a new one + * from scratch. + */ bool recreateLockFile = false; - /* Whether to update the lock file at all. If set to false, if any - change to the lock file is needed (e.g. when an input has been - added to flake.nix), you get a fatal error. */ + /** + * Whether to update the lock file at all. If set to false, if any + * change to the lock file is needed (e.g. when an input has been + * added to flake.nix), you get a fatal error. + */ bool updateLockFile = true; - /* Whether to write the lock file to disk. If set to true, if the - any changes to the lock file are needed and the flake is not - writable (i.e. is not a local Git working tree or similar), you - get a fatal error. If set to false, Nix will use the modified - lock file in memory only, without writing it to disk. */ + /** + * Whether to write the lock file to disk. If set to true, if the + * any changes to the lock file are needed and the flake is not + * writable (i.e. is not a local Git working tree or similar), you + * get a fatal error. If set to false, Nix will use the modified + * lock file in memory only, without writing it to disk. + */ bool writeLockFile = true; - /* Whether to use the registries to lookup indirect flake - references like 'nixpkgs'. */ + /** + * Whether to use the registries to lookup indirect flake + * references like 'nixpkgs'. + */ std::optional useRegistries = std::nullopt; - /* Whether to apply flake's nixConfig attribute to the configuration */ + /** + * Whether to apply flake's nixConfig attribute to the configuration + */ bool applyNixConfig = false; - /* Whether mutable flake references (i.e. those without a Git - revision or similar) without a corresponding lock are - allowed. Mutable flake references with a lock are always - allowed. */ - bool allowMutable = true; - - /* Whether to commit changes to flake.lock. */ + /** + * Whether unlocked flake references (i.e. those without a Git + * revision or similar) without a corresponding lock are + * allowed. Unlocked flake references with a lock are always + * allowed. + */ + bool allowUnlocked = true; + + /** + * Whether to commit changes to flake.lock. + */ bool commitLockFile = false; - /* Flake inputs to be overridden. */ + /** + * The path to a lock file to read instead of the `flake.lock` file in the top-level flake + */ + std::optional referenceLockFilePath; + + /** + * The path to a lock file to write to instead of the `flake.lock` file in the top-level flake + */ + std::optional outputLockFilePath; + + /** + * Flake inputs to be overridden. + */ std::map inputOverrides; - /* Flake inputs to be updated. This means that any existing lock - for those inputs will be ignored. */ + /** + * Flake inputs to be updated. This means that any existing lock + * for those inputs will be ignored. + */ std::set inputUpdates; }; diff -Nru nix-2.8.0/src/libexpr/flake/flakeref.cc nix-2.16.1+dfsg/src/libexpr/flake/flakeref.cc --- nix-2.8.0/src/libexpr/flake/flakeref.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/flake/flakeref.cc 2023-06-06 10:55:23.000000000 +0000 @@ -176,7 +176,7 @@ parsedURL.query.insert_or_assign("shallow", "1"); return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + FlakeRef(Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), fragment); } @@ -189,7 +189,7 @@ if (!hasPrefix(path, "/")) throw BadURL("flake reference '%s' is not an absolute path", url); auto query = decodeQuery(match[2]); - path = canonPath(path + "/" + get(query, "dir").value_or("")); + path = canonPath(path + "/" + getOr(query, "dir", "")); } fetchers::Attrs attrs; @@ -208,7 +208,7 @@ input.parent = baseDir; return std::make_pair( - FlakeRef(std::move(input), get(parsedURL.query, "dir").value_or("")), + FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), fragment); } } @@ -238,4 +238,15 @@ return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; } +std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( + const std::string & url, + const std::optional & baseDir, + bool allowMissing, + bool isFlake) +{ + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake); + return {std::move(flakeRef), fragment, extendedOutputsSpec}; +} + } diff -Nru nix-2.8.0/src/libexpr/flake/flakeref.hh nix-2.16.1+dfsg/src/libexpr/flake/flakeref.hh --- nix-2.8.0/src/libexpr/flake/flakeref.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/flake/flakeref.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,8 +1,10 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" #include "fetchers.hh" +#include "outputs-spec.hh" #include @@ -12,7 +14,8 @@ typedef std::string FlakeId; -/* A flake reference specifies how to fetch a flake or raw source +/** + * A flake reference specifies how to fetch a flake or raw source * (e.g. from a Git repository). It is created from a URL-like syntax * (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ * type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local @@ -27,18 +30,21 @@ * object that fetcher generates (usually via * FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls). * - * The actual fetch not have been performed yet (i.e. a FlakeRef may + * The actual fetch may not have been performed yet (i.e. a FlakeRef may * be lazy), but the fetcher can be invoked at any time via the * FlakeRef to ensure the store is populated with this input. */ - struct FlakeRef { - /* fetcher-specific representation of the input, sufficient to - perform the fetch operation. */ + /** + * Fetcher-specific representation of the input, sufficient to + * perform the fetch operation. + */ fetchers::Input input; - /* sub-path within the fetched input that represents this input */ + /** + * sub-path within the fetched input that represents this input + */ Path subdir; bool operator==(const FlakeRef & other) const; @@ -79,4 +85,11 @@ std::optional> maybeParseFlakeRefWithFragment( const std::string & url, const std::optional & baseDir = {}); +std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( + const std::string & url, + const std::optional & baseDir = {}, + bool allowMissing = false, + bool isFlake = true); + + } diff -Nru nix-2.8.0/src/libexpr/flake/lockfile.cc nix-2.16.1+dfsg/src/libexpr/flake/lockfile.cc --- nix-2.8.0/src/libexpr/flake/lockfile.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/flake/lockfile.cc 2023-06-06 10:55:23.000000000 +0000 @@ -31,12 +31,12 @@ } LockedNode::LockedNode(const nlohmann::json & json) - : lockedRef(getFlakeRef(json, "locked", "info")) + : lockedRef(getFlakeRef(json, "locked", "info")) // FIXME: remove "info" , originalRef(getFlakeRef(json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { if (!lockedRef.input.isLocked()) - throw Error("lockfile contains mutable lock '%s'", + throw Error("lock file contains mutable lock '%s'", fetchers::attrsToJSON(lockedRef.input.toAttrs())); } @@ -49,15 +49,15 @@ { auto pos = root; - if (!pos) return {}; - for (auto & elem : path) { if (auto i = get(pos->inputs, elem)) { if (auto node = std::get_if<0>(&*i)) pos = *node; else if (auto follows = std::get_if<1>(&*i)) { - pos = findInput(*follows); - if (!pos) return {}; + if (auto p = findInput(*follows)) + pos = ref(p); + else + return {}; } } else return {}; @@ -72,7 +72,7 @@ if (version < 5 || version > 7) throw Error("lock file '%s' has unsupported version %d", path, version); - std::unordered_map> nodeMap; + std::map> nodeMap; std::function getInputs; @@ -93,12 +93,12 @@ auto jsonNode2 = nodes.find(inputKey); if (jsonNode2 == nodes.end()) throw Error("lock file references missing node '%s'", inputKey); - auto input = std::make_shared(*jsonNode2); + auto input = make_ref(*jsonNode2); k = nodeMap.insert_or_assign(inputKey, input).first; getInputs(*input, *jsonNode2); } - if (auto child = std::dynamic_pointer_cast(k->second)) - node.inputs.insert_or_assign(i.key(), child); + if (auto child = k->second.dynamic_pointer_cast()) + node.inputs.insert_or_assign(i.key(), ref(child)); else // FIXME: replace by follows node throw Error("lock file contains cycle to root node"); @@ -122,9 +122,9 @@ std::unordered_map, std::string> nodeKeys; std::unordered_set keys; - std::function node)> dumpNode; + std::function node)> dumpNode; - dumpNode = [&](std::string key, std::shared_ptr node) -> std::string + dumpNode = [&](std::string key, ref node) -> std::string { auto k = nodeKeys.find(node); if (k != nodeKeys.end()) @@ -159,10 +159,11 @@ n["inputs"] = std::move(inputs); } - if (auto lockedNode = std::dynamic_pointer_cast(node)) { + if (auto lockedNode = node.dynamic_pointer_cast()) { n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs()); n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); - if (!lockedNode->isFlake) n["flake"] = false; + if (!lockedNode->isFlake) + n["flake"] = false; } nodes[key] = std::move(n); @@ -201,13 +202,13 @@ writeFile(path, fmt("%s\n", *this)); } -bool LockFile::isImmutable() const +std::optional LockFile::isUnlocked() const { - std::unordered_set> nodes; + std::set> nodes; - std::function node)> visit; + std::function node)> visit; - visit = [&](std::shared_ptr node) + visit = [&](ref node) { if (!nodes.insert(node).second) return; for (auto & i : node->inputs) @@ -218,12 +219,13 @@ visit(root); for (auto & i : nodes) { - if (i == root) continue; - auto lockedNode = std::dynamic_pointer_cast(i); - if (lockedNode && !lockedNode->lockedRef.input.isLocked()) return false; + if (i == ref(root)) continue; + auto node = i.dynamic_pointer_cast(); + if (node && !node->lockedRef.input.isLocked()) + return node->lockedRef; } - return true; + return {}; } bool LockFile::operator ==(const LockFile & other) const @@ -232,6 +234,11 @@ return toJSON() == other.toJSON(); } +bool LockFile::operator !=(const LockFile & other) const +{ + return !(*this == other); +} + InputPath parseInputPath(std::string_view s) { InputPath path; @@ -247,12 +254,12 @@ std::map LockFile::getAllInputs() const { - std::unordered_set> done; + std::set> done; std::map res; - std::function node)> recurse; + std::function node)> recurse; - recurse = [&](const InputPath & prefix, std::shared_ptr node) + recurse = [&](const InputPath & prefix, ref node) { if (!done.insert(node).second) return; diff -Nru nix-2.8.0/src/libexpr/flake/lockfile.hh nix-2.16.1+dfsg/src/libexpr/flake/lockfile.hh --- nix-2.8.0/src/libexpr/flake/lockfile.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/flake/lockfile.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "flakeref.hh" @@ -15,19 +16,23 @@ struct LockedNode; -/* A node in the lock file. It has outgoing edges to other nodes (its - inputs). Only the root node has this type; all other nodes have - type LockedNode. */ +/** + * A node in the lock file. It has outgoing edges to other nodes (its + * inputs). Only the root node has this type; all other nodes have + * type LockedNode. + */ struct Node : std::enable_shared_from_this { - typedef std::variant, InputPath> Edge; + typedef std::variant, InputPath> Edge; std::map inputs; virtual ~Node() { } }; -/* A non-root node in the lock file. */ +/** + * A non-root node in the lock file. + */ struct LockedNode : Node { FlakeRef lockedRef, originalRef; @@ -47,11 +52,13 @@ struct LockFile { - std::shared_ptr root = std::make_shared(); + ref root = make_ref(); LockFile() {}; LockFile(const nlohmann::json & json, const Path & path); + typedef std::map, std::string> KeyMap; + nlohmann::json toJSON() const; std::string to_string() const; @@ -60,9 +67,15 @@ void write(const Path & path) const; - bool isImmutable() const; + /** + * Check whether this lock file has any unlocked inputs. + */ + std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; + // Needed for old gcc versions that don't synthesize it (like gcc 8.2.2 + // that is still the default on aarch64-linux) + bool operator !=(const LockFile & other) const; std::shared_ptr findInput(const InputPath & path); @@ -70,7 +83,9 @@ static std::string diff(const LockFile & oldLocks, const LockFile & newLocks); - /* Check that every 'follows' input target exists. */ + /** + * Check that every 'follows' input target exists. + */ void check(); }; diff -Nru nix-2.8.0/src/libexpr/function-trace.hh nix-2.16.1+dfsg/src/libexpr/function-trace.hh --- nix-2.8.0/src/libexpr/function-trace.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/function-trace.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" @@ -8,7 +9,7 @@ struct FunctionCallTrace { - const Pos & pos; + const Pos pos; FunctionCallTrace(const Pos & pos); ~FunctionCallTrace(); }; diff -Nru nix-2.8.0/src/libexpr/get-drvs.cc nix-2.16.1+dfsg/src/libexpr/get-drvs.cc --- nix-2.8.0/src/libexpr/get-drvs.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/get-drvs.cc 2023-06-06 10:55:23.000000000 +0000 @@ -34,7 +34,7 @@ outputName = selectedOutputs.empty() - ? get(drv.env, "outputName").value_or("out") + ? getOr(drv.env, "outputName", "out") : *selectedOutputs.begin(); auto i = drv.outputs.find(outputName); @@ -51,7 +51,7 @@ if (name == "" && attrs) { auto i = attrs->find(state->sName); if (i == attrs->end()) throw TypeError("derivation name missing"); - name = state->forceStringNoCtx(*i->value); + name = state->forceStringNoCtx(*i->value, noPos, "while evaluating the 'name' attribute of a derivation"); } return name; } @@ -61,7 +61,7 @@ { if (system == "" && attrs) { auto i = attrs->find(state->sSystem); - system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, *i->pos); + system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, i->pos, "while evaluating the 'system' attribute of a derivation"); } return system; } @@ -71,11 +71,11 @@ { if (!drvPath && attrs) { Bindings::iterator i = attrs->find(state->sDrvPath); - PathSet context; + NixStringContext context; if (i == attrs->end()) drvPath = {std::nullopt}; else - drvPath = {state->coerceToStorePath(*i->pos, *i->value, context)}; + drvPath = {state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation")}; } return drvPath.value_or(std::nullopt); } @@ -93,9 +93,9 @@ { if (!outPath && attrs) { Bindings::iterator i = attrs->find(state->sOutPath); - PathSet context; + NixStringContext context; if (i != attrs->end()) - outPath = state->coerceToStorePath(*i->pos, *i->value, context); + outPath = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the output path of a derivation"); } if (!outPath) throw UnimplementedError("CA derivations are not yet supported"); @@ -109,46 +109,59 @@ /* Get the ‘outputs’ list. */ Bindings::iterator i; if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) { - state->forceList(*i->value, *i->pos); + state->forceList(*i->value, i->pos, "while evaluating the 'outputs' attribute of a derivation"); /* For each output... */ for (auto elem : i->value->listItems()) { - std::string output(state->forceStringNoCtx(*elem, *i->pos)); + std::string output(state->forceStringNoCtx(*elem, i->pos, "while evaluating the name of an output of a derivation")); if (withPaths) { /* Evaluate the corresponding set. */ Bindings::iterator out = attrs->find(state->symbols.create(output)); if (out == attrs->end()) continue; // FIXME: throw error? - state->forceAttrs(*out->value, *i->pos); + state->forceAttrs(*out->value, i->pos, "while evaluating an output of a derivation"); /* And evaluate its ‘outPath’ attribute. */ Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? - PathSet context; - outputs.emplace(output, state->coerceToStorePath(*outPath->pos, *outPath->value, context)); + NixStringContext context; + outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation")); } else outputs.emplace(output, std::nullopt); } } else outputs.emplace("out", withPaths ? std::optional{queryOutPath()} : std::nullopt); } + if (!onlyOutputsToInstall || !attrs) return outputs; - /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ - const Value * outTI = queryMeta("outputsToInstall"); - if (!outTI) return outputs; - const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); - /* ^ this shows during `nix-env -i` right under the bad derivation */ - if (!outTI->isList()) throw errMsg; - Outputs result; - for (auto elem : outTI->listItems()) { - if (elem->type() != nString) throw errMsg; - auto out = outputs.find(elem->string.s); - if (out == outputs.end()) throw errMsg; + Bindings::iterator i; + if (attrs && (i = attrs->find(state->sOutputSpecified)) != attrs->end() && state->forceBool(*i->value, i->pos, "while evaluating the 'outputSpecified' attribute of a derivation")) { + Outputs result; + auto out = outputs.find(queryOutputName()); + if (out == outputs.end()) + throw Error("derivation does not have output '%s'", queryOutputName()); result.insert(*out); + return result; + } + + else { + /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ + const Value * outTI = queryMeta("outputsToInstall"); + if (!outTI) return outputs; + auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); + /* ^ this shows during `nix-env -i` right under the bad derivation */ + if (!outTI->isList()) throw errMsg; + Outputs result; + for (auto elem : outTI->listItems()) { + if (elem->type() != nString) throw errMsg; + auto out = outputs.find(elem->string.s); + if (out == outputs.end()) throw errMsg; + result.insert(*out); + } + return result; } - return result; } @@ -156,7 +169,7 @@ { if (outputName == "" && attrs) { Bindings::iterator i = attrs->find(state->sOutputName); - outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : ""; + outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value, noPos, "while evaluating the output name of a derivation") : ""; } return outputName; } @@ -168,7 +181,7 @@ if (!attrs) return 0; Bindings::iterator a = attrs->find(state->sMeta); if (a == attrs->end()) return 0; - state->forceAttrs(*a->value, *a->pos); + state->forceAttrs(*a->value, a->pos, "while evaluating the 'meta' attribute of a derivation"); meta = a->value->attrs; return meta; } @@ -179,7 +192,7 @@ StringSet res; if (!getMeta()) return res; for (auto & i : *meta) - res.insert(i.name); + res.emplace(state->symbols[i.name]); return res; } @@ -269,7 +282,7 @@ { getMeta(); auto attrs = state->buildBindings(1 + (meta ? meta->size() : 0)); - Symbol sym = state->symbols.create(name); + auto sym = state->symbols.create(name); if (meta) for (auto i : *meta) if (i.name != sym) @@ -356,11 +369,11 @@ there are names clashes between derivations, the derivation bound to the attribute with the "lower" name should take precedence). */ - for (auto & i : v.attrs->lexicographicOrder()) { - debug("evaluating attribute '%1%'", i->name); - if (!std::regex_match(std::string(i->name), attrRegex)) + for (auto & i : v.attrs->lexicographicOrder(state.symbols)) { + debug("evaluating attribute '%1%'", state.symbols[i->name]); + if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex)) continue; - std::string pathPrefix2 = addToPath(pathPrefix, i->name); + std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]); if (combineChannels) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { @@ -369,7 +382,7 @@ `recurseForDerivations = true' attribute. */ if (i->value->type() == nAttrs) { Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations); - if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos)) + if (j != i->value->attrs->end() && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`")) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); } } diff -Nru nix-2.8.0/src/libexpr/get-drvs.hh nix-2.16.1+dfsg/src/libexpr/get-drvs.hh --- nix-2.8.0/src/libexpr/get-drvs.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/get-drvs.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" #include "path.hh" @@ -25,7 +26,10 @@ mutable std::string outputName; Outputs outputs; - bool failed = false; // set if we get an AssertionError + /** + * Set if we get an AssertionError + */ + bool failed = false; Bindings * attrs = nullptr, * meta = nullptr; @@ -34,7 +38,10 @@ bool checkMeta(Value & v); public: - std::string attrPath; /* path towards the derivation */ + /** + * path towards the derivation + */ + std::string attrPath; DrvInfo(EvalState & state) : state(&state) { }; DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs); @@ -46,8 +53,10 @@ StorePath requireDrvPath() const; StorePath queryOutPath() const; std::string queryOutputName() const; - /** Return the unordered map of output names to (optional) output paths. - * The "outputs to install" are determined by `meta.outputsToInstall`. */ + /** + * Return the unordered map of output names to (optional) output paths. + * The "outputs to install" are determined by `meta.outputsToInstall`. + */ Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false); StringSet queryMetaNames(); @@ -73,14 +82,16 @@ #if HAVE_BOEHMGC -typedef std::list > DrvInfos; +typedef std::list> DrvInfos; #else typedef std::list DrvInfos; #endif -/* If value `v' denotes a derivation, return a DrvInfo object - describing it. Otherwise return nothing. */ +/** + * If value `v` denotes a derivation, return a DrvInfo object + * describing it. Otherwise return nothing. + */ std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); diff -Nru nix-2.8.0/src/libexpr/json-to-value.hh nix-2.16.1+dfsg/src/libexpr/json-to-value.hh --- nix-2.8.0/src/libexpr/json-to-value.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/json-to-value.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" diff -Nru nix-2.8.0/src/libexpr/lexer.l nix-2.16.1+dfsg/src/libexpr/lexer.l --- nix-2.8.0/src/libexpr/lexer.l 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/lexer.l 2023-06-06 10:55:23.000000000 +0000 @@ -28,9 +28,9 @@ namespace nix { -static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data) +static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) { - return Pos(data->origin, data->file, loc.first_line, loc.first_column); + return data->state.positions.add(data->origin, loc.first_line, loc.first_column); } #define CUR_POS makeCurPos(*yylloc, data) @@ -155,7 +155,7 @@ } catch (const boost::bad_lexical_cast &) { throw ParseError({ .msg = hintfmt("invalid integer '%1%'", yytext), - .errPos = CUR_POS, + .errPos = data->state.positions[CUR_POS], }); } return INT; @@ -165,7 +165,7 @@ if (errno != 0) throw ParseError({ .msg = hintfmt("invalid float '%1%'", yytext), - .errPos = CUR_POS, + .errPos = data->state.positions[CUR_POS], }); return FLOAT; } @@ -198,7 +198,7 @@ (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered. This is technically invalid, but we leave the problem to the parser who fails with exact location. */ - return STR; + return EOF; } \'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } @@ -294,7 +294,7 @@ <> { throw ParseError({ .msg = hintfmt("path has a trailing slash"), - .errPos = CUR_POS, + .errPos = data->state.positions[CUR_POS], }); } diff -Nru nix-2.8.0/src/libexpr/local.mk nix-2.16.1+dfsg/src/libexpr/local.mk --- nix-2.8.0/src/libexpr/local.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -6,6 +6,7 @@ libexpr_SOURCES := \ $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) \ $(wildcard $(d)/primops/*.cc) \ $(wildcard $(d)/flake/*.cc) \ $(d)/lexer-tab.cc \ @@ -37,9 +38,13 @@ $(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) +$(foreach i, $(wildcard src/libexpr/value/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644))) $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh + +src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = diff -Nru nix-2.8.0/src/libexpr/nixexpr.cc nix-2.16.1+dfsg/src/libexpr/nixexpr.cc --- nix-2.8.0/src/libexpr/nixexpr.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/nixexpr.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,134 +1,165 @@ #include "nixexpr.hh" #include "derivations.hh" +#include "eval.hh" +#include "symbol-table.hh" #include "util.hh" +#include "print.hh" #include - namespace nix { - -/* Displaying abstract syntax trees. */ - -std::ostream & operator << (std::ostream & str, const Expr & e) +struct PosAdapter : AbstractPos { - e.show(str); - return str; -} + Pos::Origin origin; -static void showString(std::ostream & str, std::string_view s) -{ - str << '"'; - for (auto c : s) - if (c == '"' || c == '\\' || c == '$') str << "\\" << c; - else if (c == '\n') str << "\\n"; - else if (c == '\r') str << "\\r"; - else if (c == '\t') str << "\\t"; - else str << c; - str << '"'; -} - -static void showId(std::ostream & str, std::string_view s) -{ - if (s.empty()) - str << "\"\""; - else if (s == "if") // FIXME: handle other keywords - str << '"' << s << '"'; - else { - char c = s[0]; - if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { - showString(str, s); - return; - } - for (auto c : s) - if (!((c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || - c == '_' || c == '\'' || c == '-')) { - showString(str, s); - return; + PosAdapter(Pos::Origin origin) + : origin(std::move(origin)) + { + } + + std::optional getSource() const override + { + return std::visit(overloaded { + [](const Pos::none_tag &) -> std::optional { + return std::nullopt; + }, + [](const Pos::Stdin & s) -> std::optional { + // Get rid of the null terminators added by the parser. + return std::string(s.source->c_str()); + }, + [](const Pos::String & s) -> std::optional { + // Get rid of the null terminators added by the parser. + return std::string(s.source->c_str()); + }, + [](const SourcePath & path) -> std::optional { + try { + return path.readFile(); + } catch (Error &) { + return std::nullopt; + } } - str << s; + }, origin); + } + + void print(std::ostream & out) const override + { + std::visit(overloaded { + [&](const Pos::none_tag &) { out << "«none»"; }, + [&](const Pos::Stdin &) { out << "«stdin»"; }, + [&](const Pos::String & s) { out << "«string»"; }, + [&](const SourcePath & path) { out << path; } + }, origin); } +}; + +Pos::operator std::shared_ptr() const +{ + auto pos = std::make_shared(origin); + pos->line = line; + pos->column = column; + return pos; } -std::ostream & operator << (std::ostream & str, const Symbol & sym) +// FIXME: remove, because *symbols* are abstract and do not have a single +// textual representation; see printIdentifier() +std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) { - showId(str, *sym.s); - return str; + std::string_view s = symbol; + return printIdentifier(str, s); } -void Expr::show(std::ostream & str) const +void Expr::show(const SymbolTable & symbols, std::ostream & str) const { abort(); } -void ExprInt::show(std::ostream & str) const +void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const { str << n; } -void ExprFloat::show(std::ostream & str) const +void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const { str << nf; } -void ExprString::show(std::ostream & str) const +void ExprString::show(const SymbolTable & symbols, std::ostream & str) const { - showString(str, s); + printLiteralString(str, s); } -void ExprPath::show(std::ostream & str) const +void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const { str << s; } -void ExprVar::show(std::ostream & str) const +void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const { - str << name; + str << symbols[name]; } -void ExprSelect::show(std::ostream & str) const +void ExprSelect::show(const SymbolTable & symbols, std::ostream & str) const { - str << "(" << *e << ")." << showAttrPath(attrPath); - if (def) str << " or (" << *def << ")"; + str << "("; + e->show(symbols, str); + str << ")." << showAttrPath(symbols, attrPath); + if (def) { + str << " or ("; + def->show(symbols, str); + str << ")"; + } } -void ExprOpHasAttr::show(std::ostream & str) const +void ExprOpHasAttr::show(const SymbolTable & symbols, std::ostream & str) const { - str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")"; + str << "(("; + e->show(symbols, str); + str << ") ? " << showAttrPath(symbols, attrPath) << ")"; } -void ExprAttrs::show(std::ostream & str) const +void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const { if (recursive) str << "rec "; str << "{ "; typedef const decltype(attrs)::value_type * Attr; std::vector sorted; for (auto & i : attrs) sorted.push_back(&i); - std::sort(sorted.begin(), sorted.end(), [](Attr a, Attr b) { - return (const std::string &) a->first < (const std::string &) b->first; - }); + std::sort(sorted.begin(), sorted.end(), [&](Attr a, Attr b) { + std::string_view sa = symbols[a->first], sb = symbols[b->first]; + return sa < sb; + }); for (auto & i : sorted) { if (i->second.inherited) - str << "inherit " << i->first << " " << "; "; - else - str << i->first << " = " << *i->second.e << "; "; + str << "inherit " << symbols[i->first] << " " << "; "; + else { + str << symbols[i->first] << " = "; + i->second.e->show(symbols, str); + str << "; "; + } + } + for (auto & i : dynamicAttrs) { + str << "\"${"; + i.nameExpr->show(symbols, str); + str << "}\" = "; + i.valueExpr->show(symbols, str); + str << "; "; } - for (auto & i : dynamicAttrs) - str << "\"${" << *i.nameExpr << "}\" = " << *i.valueExpr << "; "; str << "}"; } -void ExprList::show(std::ostream & str) const +void ExprList::show(const SymbolTable & symbols, std::ostream & str) const { str << "[ "; - for (auto & i : elems) - str << "(" << *i << ") "; + for (auto & i : elems) { + str << "("; + i->show(symbols, str); + str << ") "; + } str << "]"; } -void ExprLambda::show(std::ostream & str) const +void ExprLambda::show(const SymbolTable & symbols, std::ostream & str) const { str << "("; if (hasFormals()) { @@ -136,74 +167,100 @@ bool first = true; for (auto & i : formals->formals) { if (first) first = false; else str << ", "; - str << i.name; - if (i.def) str << " ? " << *i.def; + str << symbols[i.name]; + if (i.def) { + str << " ? "; + i.def->show(symbols, str); + } } if (formals->ellipsis) { if (!first) str << ", "; str << "..."; } str << " }"; - if (!arg.empty()) str << " @ "; + if (arg) str << " @ "; } - if (!arg.empty()) str << arg; - str << ": " << *body << ")"; + if (arg) str << symbols[arg]; + str << ": "; + body->show(symbols, str); + str << ")"; } -void ExprCall::show(std::ostream & str) const +void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const { - str << '(' << *fun; + str << '('; + fun->show(symbols, str); for (auto e : args) { str << ' '; - str << *e; + e->show(symbols, str); } str << ')'; } -void ExprLet::show(std::ostream & str) const +void ExprLet::show(const SymbolTable & symbols, std::ostream & str) const { str << "(let "; for (auto & i : attrs->attrs) if (i.second.inherited) { - str << "inherit " << i.first << "; "; + str << "inherit " << symbols[i.first] << "; "; + } + else { + str << symbols[i.first] << " = "; + i.second.e->show(symbols, str); + str << "; "; } - else - str << i.first << " = " << *i.second.e << "; "; - str << "in " << *body << ")"; + str << "in "; + body->show(symbols, str); + str << ")"; } -void ExprWith::show(std::ostream & str) const +void ExprWith::show(const SymbolTable & symbols, std::ostream & str) const { - str << "(with " << *attrs << "; " << *body << ")"; + str << "(with "; + attrs->show(symbols, str); + str << "; "; + body->show(symbols, str); + str << ")"; } -void ExprIf::show(std::ostream & str) const +void ExprIf::show(const SymbolTable & symbols, std::ostream & str) const { - str << "(if " << *cond << " then " << *then << " else " << *else_ << ")"; + str << "(if "; + cond->show(symbols, str); + str << " then "; + then->show(symbols, str); + str << " else "; + else_->show(symbols, str); + str << ")"; } -void ExprAssert::show(std::ostream & str) const +void ExprAssert::show(const SymbolTable & symbols, std::ostream & str) const { - str << "assert " << *cond << "; " << *body; + str << "assert "; + cond->show(symbols, str); + str << "; "; + body->show(symbols, str); } -void ExprOpNot::show(std::ostream & str) const +void ExprOpNot::show(const SymbolTable & symbols, std::ostream & str) const { - str << "(! " << *e << ")"; + str << "(! "; + e->show(symbols, str); + str << ")"; } -void ExprConcatStrings::show(std::ostream & str) const +void ExprConcatStrings::show(const SymbolTable & symbols, std::ostream & str) const { bool first = true; str << "("; for (auto & i : *es) { if (first) first = false; else str << " + "; - str << *i.second; + i.second->show(symbols, str); } str << ")"; } -void ExprPos::show(std::ostream & str) const +void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const { str << "__curPos"; } @@ -211,78 +268,75 @@ std::ostream & operator << (std::ostream & str, const Pos & pos) { - if (!pos) + if (auto pos2 = (std::shared_ptr) pos) { + str << *pos2; + } else str << "undefined position"; - else - { - auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%"); - switch (pos.origin) { - case foFile: - f % (const std::string &) pos.file; - break; - case foStdin: - case foString: - f % "(string)"; - break; - default: - throw Error("unhandled Pos origin!"); - } - str << (f % pos.line % pos.column).str(); - } return str; } -std::string showAttrPath(const AttrPath & attrPath) +std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) { std::ostringstream out; bool first = true; for (auto & i : attrPath) { if (!first) out << '.'; else first = false; - if (i.symbol.set()) - out << i.symbol; - else - out << "\"${" << *i.expr << "}\""; + if (i.symbol) + out << symbols[i.symbol]; + else { + out << "\"${"; + i.expr->show(symbols, out); + out << "}\""; + } } return out.str(); } -Pos noPos; - - /* Computing levels/displacements for variables. */ -void Expr::bindVars(const StaticEnv & env) +void Expr::bindVars(EvalState & es, const std::shared_ptr & env) { abort(); } -void ExprInt::bindVars(const StaticEnv & env) +void ExprInt::bindVars(EvalState & es, const std::shared_ptr & env) { + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); } -void ExprFloat::bindVars(const StaticEnv & env) +void ExprFloat::bindVars(EvalState & es, const std::shared_ptr & env) { + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); } -void ExprString::bindVars(const StaticEnv & env) +void ExprString::bindVars(EvalState & es, const std::shared_ptr & env) { + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); } -void ExprPath::bindVars(const StaticEnv & env) +void ExprPath::bindVars(EvalState & es, const std::shared_ptr & env) { + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); } -void ExprVar::bindVars(const StaticEnv & env) +void ExprVar::bindVars(EvalState & es, const std::shared_ptr & env) { + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); + /* Check whether the variable appears in the environment. If so, set its level and displacement. */ const StaticEnv * curEnv; Level level; int withLevel = -1; - for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) { + for (curEnv = env.get(), level = 0; curEnv; curEnv = curEnv->up, level++) { if (curEnv->isWith) { if (withLevel == -1) withLevel = level; } else { @@ -301,176 +355,222 @@ "undefined variable" error now. */ if (withLevel == -1) throw UndefinedVarError({ - .msg = hintfmt("undefined variable '%1%'", name), - .errPos = pos + .msg = hintfmt("undefined variable '%1%'", es.symbols[name]), + .errPos = es.positions[pos] }); fromWith = true; this->level = withLevel; } -void ExprSelect::bindVars(const StaticEnv & env) +void ExprSelect::bindVars(EvalState & es, const std::shared_ptr & env) { - e->bindVars(env); - if (def) def->bindVars(env); + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); + + e->bindVars(es, env); + if (def) def->bindVars(es, env); for (auto & i : attrPath) - if (!i.symbol.set()) - i.expr->bindVars(env); + if (!i.symbol) + i.expr->bindVars(es, env); } -void ExprOpHasAttr::bindVars(const StaticEnv & env) +void ExprOpHasAttr::bindVars(EvalState & es, const std::shared_ptr & env) { - e->bindVars(env); + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); + + e->bindVars(es, env); for (auto & i : attrPath) - if (!i.symbol.set()) - i.expr->bindVars(env); + if (!i.symbol) + i.expr->bindVars(es, env); } -void ExprAttrs::bindVars(const StaticEnv & env) +void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr & env) { - const StaticEnv * dynamicEnv = &env; - StaticEnv newEnv(false, &env, recursive ? attrs.size() : 0); + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); if (recursive) { - dynamicEnv = &newEnv; + auto newEnv = std::make_shared(false, env.get(), recursive ? attrs.size() : 0); Displacement displ = 0; for (auto & i : attrs) - newEnv.vars.emplace_back(i.first, i.second.displ = displ++); + newEnv->vars.emplace_back(i.first, i.second.displ = displ++); // No need to sort newEnv since attrs is in sorted order. for (auto & i : attrs) - i.second.e->bindVars(i.second.inherited ? env : newEnv); - } + i.second.e->bindVars(es, i.second.inherited ? env : newEnv); - else + for (auto & i : dynamicAttrs) { + i.nameExpr->bindVars(es, newEnv); + i.valueExpr->bindVars(es, newEnv); + } + } + else { for (auto & i : attrs) - i.second.e->bindVars(env); + i.second.e->bindVars(es, env); - for (auto & i : dynamicAttrs) { - i.nameExpr->bindVars(*dynamicEnv); - i.valueExpr->bindVars(*dynamicEnv); + for (auto & i : dynamicAttrs) { + i.nameExpr->bindVars(es, env); + i.valueExpr->bindVars(es, env); + } } } -void ExprList::bindVars(const StaticEnv & env) +void ExprList::bindVars(EvalState & es, const std::shared_ptr & env) { + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); + for (auto & i : elems) - i->bindVars(env); + i->bindVars(es, env); } -void ExprLambda::bindVars(const StaticEnv & env) +void ExprLambda::bindVars(EvalState & es, const std::shared_ptr & env) { - StaticEnv newEnv( - false, &env, + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); + + auto newEnv = std::make_shared( + false, env.get(), (hasFormals() ? formals->formals.size() : 0) + - (arg.empty() ? 0 : 1)); + (!arg ? 0 : 1)); Displacement displ = 0; - if (!arg.empty()) newEnv.vars.emplace_back(arg, displ++); + if (arg) newEnv->vars.emplace_back(arg, displ++); if (hasFormals()) { for (auto & i : formals->formals) - newEnv.vars.emplace_back(i.name, displ++); + newEnv->vars.emplace_back(i.name, displ++); - newEnv.sort(); + newEnv->sort(); for (auto & i : formals->formals) - if (i.def) i.def->bindVars(newEnv); + if (i.def) i.def->bindVars(es, newEnv); } - body->bindVars(newEnv); + body->bindVars(es, newEnv); } -void ExprCall::bindVars(const StaticEnv & env) +void ExprCall::bindVars(EvalState & es, const std::shared_ptr & env) { - fun->bindVars(env); + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); + + fun->bindVars(es, env); for (auto e : args) - e->bindVars(env); + e->bindVars(es, env); } -void ExprLet::bindVars(const StaticEnv & env) +void ExprLet::bindVars(EvalState & es, const std::shared_ptr & env) { - StaticEnv newEnv(false, &env, attrs->attrs.size()); + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); + + auto newEnv = std::make_shared(false, env.get(), attrs->attrs.size()); Displacement displ = 0; for (auto & i : attrs->attrs) - newEnv.vars.emplace_back(i.first, i.second.displ = displ++); + newEnv->vars.emplace_back(i.first, i.second.displ = displ++); // No need to sort newEnv since attrs->attrs is in sorted order. for (auto & i : attrs->attrs) - i.second.e->bindVars(i.second.inherited ? env : newEnv); + i.second.e->bindVars(es, i.second.inherited ? env : newEnv); - body->bindVars(newEnv); + body->bindVars(es, newEnv); } -void ExprWith::bindVars(const StaticEnv & env) +void ExprWith::bindVars(EvalState & es, const std::shared_ptr & env) { + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); + /* Does this `with' have an enclosing `with'? If so, record its level so that `lookupVar' can look up variables in the previous `with' if this one doesn't contain the desired attribute. */ const StaticEnv * curEnv; Level level; prevWith = 0; - for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++) + for (curEnv = env.get(), level = 1; curEnv; curEnv = curEnv->up, level++) if (curEnv->isWith) { prevWith = level; break; } - attrs->bindVars(env); - StaticEnv newEnv(true, &env); - body->bindVars(newEnv); + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); + + attrs->bindVars(es, env); + auto newEnv = std::make_shared(true, env.get()); + body->bindVars(es, newEnv); } -void ExprIf::bindVars(const StaticEnv & env) +void ExprIf::bindVars(EvalState & es, const std::shared_ptr & env) { - cond->bindVars(env); - then->bindVars(env); - else_->bindVars(env); + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); + + cond->bindVars(es, env); + then->bindVars(es, env); + else_->bindVars(es, env); } -void ExprAssert::bindVars(const StaticEnv & env) +void ExprAssert::bindVars(EvalState & es, const std::shared_ptr & env) { - cond->bindVars(env); - body->bindVars(env); + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); + + cond->bindVars(es, env); + body->bindVars(es, env); } -void ExprOpNot::bindVars(const StaticEnv & env) +void ExprOpNot::bindVars(EvalState & es, const std::shared_ptr & env) { - e->bindVars(env); + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); + + e->bindVars(es, env); } -void ExprConcatStrings::bindVars(const StaticEnv & env) +void ExprConcatStrings::bindVars(EvalState & es, const std::shared_ptr & env) { - for (auto & i : *es) - i.second->bindVars(env); + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); + + for (auto & i : *this->es) + i.second->bindVars(es, env); } -void ExprPos::bindVars(const StaticEnv & env) +void ExprPos::bindVars(EvalState & es, const std::shared_ptr & env) { + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); } /* Storing function names. */ -void Expr::setName(Symbol & name) +void Expr::setName(Symbol name) { } -void ExprLambda::setName(Symbol & name) +void ExprLambda::setName(Symbol name) { this->name = name; body->setName(name); } -std::string ExprLambda::showNamePos() const +std::string ExprLambda::showNamePos(const EvalState & state) const { - return fmt("%1% at %2%", name.set() ? "'" + (std::string) name + "'" : "anonymous function", pos); + std::string id(name + ? concatStrings("'", state.symbols[name], "'") + : "anonymous function"); + return fmt("%1% at %2%", id, state.positions[pos]); } @@ -480,8 +580,7 @@ size_t SymbolTable::totalSize() const { size_t n = 0; - for (auto & i : store) - n += i.size(); + dump([&] (const std::string & s) { n += s.size(); }); return n; } diff -Nru nix-2.8.0/src/libexpr/nixexpr.hh nix-2.16.1+dfsg/src/libexpr/nixexpr.hh --- nix-2.8.0/src/libexpr/nixexpr.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/nixexpr.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,9 +1,13 @@ #pragma once +///@file + +#include +#include #include "value.hh" #include "symbol-table.hh" #include "error.hh" - +#include "chunked-vector.hh" namespace nix { @@ -18,37 +22,109 @@ MakeError(MissingArgumentError, EvalError); MakeError(RestrictedPathError, Error); - -/* Position objects. */ - +/** + * Position objects. + */ struct Pos { - Symbol file; uint32_t line; - FileOrigin origin:2; - uint32_t column:30; - Pos() : line(0), origin(foString), column(0) { }; - Pos(FileOrigin origin, const Symbol & file, uint32_t line, uint32_t column) - : file(file), line(line), origin(origin), column(column) { }; - operator bool() const + uint32_t column; + + struct none_tag { }; + struct Stdin { ref source; }; + struct String { ref source; }; + + typedef std::variant Origin; + + Origin origin; + + explicit operator bool() const { return line > 0; } + + operator std::shared_ptr() const; +}; + +class PosIdx { + friend class PosTable; + +private: + uint32_t id; + + explicit PosIdx(uint32_t id): id(id) {} + +public: + PosIdx() : id(0) {} + + explicit operator bool() const { return id > 0; } + + bool operator <(const PosIdx other) const { return id < other.id; } + + bool operator ==(const PosIdx other) const { return id == other.id; } + + bool operator !=(const PosIdx other) const { return id != other.id; } +}; + +class PosTable +{ +public: + class Origin { + friend PosTable; + private: + // must always be invalid by default, add() replaces this with the actual value. + // subsequent add() calls use this index as a token to quickly check whether the + // current origins.back() can be reused or not. + mutable uint32_t idx = std::numeric_limits::max(); + + // Used for searching in PosTable::[]. + explicit Origin(uint32_t idx): idx(idx), origin{Pos::none_tag()} {} + + public: + const Pos::Origin origin; + + Origin(Pos::Origin origin): origin(origin) {} + }; + + struct Offset { + uint32_t line, column; + }; + +private: + std::vector origins; + ChunkedVector offsets; + +public: + PosTable(): offsets(1024) + { + origins.reserve(1024); + } + + PosIdx add(const Origin & origin, uint32_t line, uint32_t column) { - return line != 0; + const auto idx = offsets.add({line, column}).second; + if (origins.empty() || origins.back().idx != origin.idx) { + origin.idx = idx; + origins.push_back(origin); + } + return PosIdx(idx + 1); } - bool operator < (const Pos & p2) const + Pos operator[](PosIdx p) const { - if (!line) return p2.line; - if (!p2.line) return false; - int d = ((const std::string &) file).compare((const std::string &) p2.file); - if (d < 0) return true; - if (d > 0) return false; - if (line < p2.line) return true; - if (line > p2.line) return false; - return column < p2.column; + if (p.id == 0 || p.id > offsets.size()) + return {}; + const auto idx = p.id - 1; + /* we want the last key <= idx, so we'll take prev(first key > idx). + this is guaranteed to never rewind origin.begin because the first + key is always 0. */ + const auto pastOrigin = std::upper_bound( + origins.begin(), origins.end(), Origin(idx), + [] (const auto & a, const auto & b) { return a.idx < b.idx; }); + const auto origin = *std::prev(pastOrigin); + const auto offset = offsets[idx]; + return {offset.line, offset.column, origin.origin}; } }; -extern Pos noPos; +inline PosIdx noPos = {}; std::ostream & operator << (std::ostream & str, const Pos & pos); @@ -59,18 +135,20 @@ struct StaticEnv; -/* An attribute path is a sequence of attribute names. */ +/** + * An attribute path is a sequence of attribute names. + */ struct AttrName { Symbol symbol; Expr * expr; - AttrName(const Symbol & s) : symbol(s) {}; + AttrName(Symbol s) : symbol(s) {}; AttrName(Expr * e) : expr(e) {}; }; typedef std::vector AttrPath; -std::string showAttrPath(const AttrPath & attrPath); +std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath); /* Abstract syntax of Nix expressions. */ @@ -78,27 +156,26 @@ struct Expr { virtual ~Expr() { }; - virtual void show(std::ostream & str) const; - virtual void bindVars(const StaticEnv & env); + virtual void show(const SymbolTable & symbols, std::ostream & str) const; + virtual void bindVars(EvalState & es, const std::shared_ptr & env); virtual void eval(EvalState & state, Env & env, Value & v); virtual Value * maybeThunk(EvalState & state, Env & env); - virtual void setName(Symbol & name); + virtual void setName(Symbol name); + virtual PosIdx getPos() const { return noPos; } }; -std::ostream & operator << (std::ostream & str, const Expr & e); - #define COMMON_METHODS \ - void show(std::ostream & str) const; \ - void eval(EvalState & state, Env & env, Value & v); \ - void bindVars(const StaticEnv & env); + void show(const SymbolTable & symbols, std::ostream & str) const override; \ + void eval(EvalState & state, Env & env, Value & v) override; \ + void bindVars(EvalState & es, const std::shared_ptr & env) override; struct ExprInt : Expr { NixInt n; Value v; ExprInt(NixInt n) : n(n) { v.mkInt(n); }; + Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); }; struct ExprFloat : Expr @@ -106,17 +183,17 @@ NixFloat nf; Value v; ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); }; + Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); }; struct ExprString : Expr { std::string s; Value v; - ExprString(std::string s) : s(std::move(s)) { v.mkString(this->s.data()); }; + ExprString(std::string &&s) : s(std::move(s)) { v.mkString(this->s.data()); }; + Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); }; struct ExprPath : Expr @@ -124,8 +201,8 @@ std::string s; Value v; ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); }; + Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); }; typedef uint32_t Level; @@ -133,35 +210,37 @@ struct ExprVar : Expr { - Pos pos; + PosIdx pos; Symbol name; /* Whether the variable comes from an environment (e.g. a rec, let or function argument) or from a "with". */ bool fromWith; - /* In the former case, the value is obtained by going `level' + /* In the former case, the value is obtained by going `level` levels up from the current environment and getting the - `displ'th value in that environment. In the latter case, the - value is obtained by getting the attribute named `name' from - the set stored in the environment that is `level' levels up + `displ`th value in that environment. In the latter case, the + value is obtained by getting the attribute named `name` from + the set stored in the environment that is `level` levels up from the current one.*/ Level level; Displacement displ; - ExprVar(const Symbol & name) : name(name) { }; - ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { }; + ExprVar(Symbol name) : name(name) { }; + ExprVar(const PosIdx & pos, Symbol name) : pos(pos), name(name) { }; + Value * maybeThunk(EvalState & state, Env & env) override; + PosIdx getPos() const override { return pos; } COMMON_METHODS - Value * maybeThunk(EvalState & state, Env & env); }; struct ExprSelect : Expr { - Pos pos; + PosIdx pos; Expr * e, * def; AttrPath attrPath; - ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { }; - ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; + ExprSelect(const PosIdx & pos, Expr * e, const AttrPath && attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; + ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; + PosIdx getPos() const override { return pos; } COMMON_METHODS }; @@ -169,20 +248,21 @@ { Expr * e; AttrPath attrPath; - ExprOpHasAttr(Expr * e, const AttrPath & attrPath) : e(e), attrPath(attrPath) { }; + ExprOpHasAttr(Expr * e, const AttrPath && attrPath) : e(e), attrPath(std::move(attrPath)) { }; + PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; struct ExprAttrs : Expr { bool recursive; - Pos pos; + PosIdx pos; struct AttrDef { bool inherited; Expr * e; - Pos pos; + PosIdx pos; Displacement displ; // displacement - AttrDef(Expr * e, const Pos & pos, bool inherited=false) + AttrDef(Expr * e, const PosIdx & pos, bool inherited=false) : inherited(inherited), e(e), pos(pos) { }; AttrDef() { }; }; @@ -190,14 +270,15 @@ AttrDefs attrs; struct DynamicAttrDef { Expr * nameExpr, * valueExpr; - Pos pos; - DynamicAttrDef(Expr * nameExpr, Expr * valueExpr, const Pos & pos) + PosIdx pos; + DynamicAttrDef(Expr * nameExpr, Expr * valueExpr, const PosIdx & pos) : nameExpr(nameExpr), valueExpr(valueExpr), pos(pos) { }; }; typedef std::vector DynamicAttrDefs; DynamicAttrDefs dynamicAttrs; - ExprAttrs(const Pos &pos) : recursive(false), pos(pos) { }; - ExprAttrs() : recursive(false), pos(noPos) { }; + ExprAttrs(const PosIdx &pos) : recursive(false), pos(pos) { }; + ExprAttrs() : recursive(false) { }; + PosIdx getPos() const override { return pos; } COMMON_METHODS }; @@ -206,14 +287,18 @@ std::vector elems; ExprList() { }; COMMON_METHODS + + PosIdx getPos() const override + { + return elems.empty() ? noPos : elems.front()->getPos(); + } }; struct Formal { - Pos pos; + PosIdx pos; Symbol name; Expr * def; - Formal(const Pos & pos, const Symbol & name, Expr * def) : pos(pos), name(name), def(def) { }; }; struct Formals @@ -222,18 +307,20 @@ Formals_ formals; bool ellipsis; - bool has(Symbol arg) const { + bool has(Symbol arg) const + { auto it = std::lower_bound(formals.begin(), formals.end(), arg, [] (const Formal & f, const Symbol & sym) { return f.name < sym; }); return it != formals.end() && it->name == arg; } - std::vector lexicographicOrder() const + std::vector lexicographicOrder(const SymbolTable & symbols) const { std::vector result(formals.begin(), formals.end()); std::sort(result.begin(), result.end(), - [] (const Formal & a, const Formal & b) { - return std::string_view(a.name) < std::string_view(b.name); + [&] (const Formal & a, const Formal & b) { + std::string_view sa = symbols[a.name], sb = symbols[b.name]; + return sa < sb; }); return result; } @@ -241,18 +328,23 @@ struct ExprLambda : Expr { - Pos pos; + PosIdx pos; Symbol name; Symbol arg; Formals * formals; Expr * body; - ExprLambda(const Pos & pos, const Symbol & arg, Formals * formals, Expr * body) + ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body) : pos(pos), arg(arg), formals(formals), body(body) { }; - void setName(Symbol & name); - std::string showNamePos() const; + ExprLambda(PosIdx pos, Formals * formals, Expr * body) + : pos(pos), formals(formals), body(body) + { + } + void setName(Symbol name) override; + std::string showNamePos(const EvalState & state) const; inline bool hasFormals() const { return formals != nullptr; } + PosIdx getPos() const override { return pos; } COMMON_METHODS }; @@ -260,10 +352,11 @@ { Expr * fun; std::vector args; - Pos pos; - ExprCall(const Pos & pos, Expr * fun, std::vector && args) + PosIdx pos; + ExprCall(const PosIdx & pos, Expr * fun, std::vector && args) : fun(fun), args(args), pos(pos) { } + PosIdx getPos() const override { return pos; } COMMON_METHODS }; @@ -277,26 +370,29 @@ struct ExprWith : Expr { - Pos pos; + PosIdx pos; Expr * attrs, * body; size_t prevWith; - ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { }; + ExprWith(const PosIdx & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { }; + PosIdx getPos() const override { return pos; } COMMON_METHODS }; struct ExprIf : Expr { - Pos pos; + PosIdx pos; Expr * cond, * then, * else_; - ExprIf(const Pos & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { }; + ExprIf(const PosIdx & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { }; + PosIdx getPos() const override { return pos; } COMMON_METHODS }; struct ExprAssert : Expr { - Pos pos; + PosIdx pos; Expr * cond, * body; - ExprAssert(const Pos & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { }; + ExprAssert(const PosIdx & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { }; + PosIdx getPos() const override { return pos; } COMMON_METHODS }; @@ -310,19 +406,20 @@ #define MakeBinOp(name, s) \ struct name : Expr \ { \ - Pos pos; \ + PosIdx pos; \ Expr * e1, * e2; \ name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \ - name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \ - void show(std::ostream & str) const \ + name(const PosIdx & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \ + void show(const SymbolTable & symbols, std::ostream & str) const override \ { \ - str << "(" << *e1 << " " s " " << *e2 << ")"; \ + str << "("; e1->show(symbols, str); str << " " s " "; e2->show(symbols, str); str << ")"; \ } \ - void bindVars(const StaticEnv & env) \ + void bindVars(EvalState & es, const std::shared_ptr & env) override \ { \ - e1->bindVars(env); e2->bindVars(env); \ + e1->bindVars(es, env); e2->bindVars(es, env); \ } \ - void eval(EvalState & state, Env & env, Value & v); \ + void eval(EvalState & state, Env & env, Value & v) override; \ + PosIdx getPos() const override { return pos; } \ }; MakeBinOp(ExprOpEq, "==") @@ -335,18 +432,20 @@ struct ExprConcatStrings : Expr { - Pos pos; + PosIdx pos; bool forceString; - std::vector > * es; - ExprConcatStrings(const Pos & pos, bool forceString, std::vector > * es) + std::vector> * es; + ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector> * es) : pos(pos), forceString(forceString), es(es) { }; + PosIdx getPos() const override { return pos; } COMMON_METHODS }; struct ExprPos : Expr { - Pos pos; - ExprPos(const Pos & pos) : pos(pos) { }; + PosIdx pos; + ExprPos(const PosIdx & pos) : pos(pos) { }; + PosIdx getPos() const override { return pos; } COMMON_METHODS }; @@ -384,7 +483,7 @@ vars.erase(it, end); } - Vars::const_iterator find(const Symbol & name) const + Vars::const_iterator find(Symbol name) const { Vars::value_type key(name, 0); auto i = std::lower_bound(vars.begin(), vars.end(), key); diff -Nru nix-2.8.0/src/libexpr/nix-expr.pc.in nix-2.16.1+dfsg/src/libexpr/nix-expr.pc.in --- nix-2.8.0/src/libexpr/nix-expr.pc.in 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/nix-expr.pc.in 2023-06-06 10:55:23.000000000 +0000 @@ -7,4 +7,4 @@ Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff -Nru nix-2.8.0/src/libexpr/parser.y nix-2.16.1+dfsg/src/libexpr/parser.y --- nix-2.8.0/src/libexpr/parser.y 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/parser.y 2023-06-06 10:55:23.000000000 +0000 @@ -31,14 +31,9 @@ EvalState & state; SymbolTable & symbols; Expr * result; - Path basePath; - Symbol file; - FileOrigin origin; + SourcePath basePath; + PosTable::Origin origin; std::optional error; - ParseData(EvalState & state) - : state(state) - , symbols(state.symbols) - { }; }; struct ParserFormals { @@ -77,26 +72,26 @@ namespace nix { -static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) +static void dupAttr(const EvalState & state, const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos) { throw ParseError({ .msg = hintfmt("attribute '%1%' already defined at %2%", - showAttrPath(attrPath), prevPos), - .errPos = pos + showAttrPath(state.symbols, attrPath), state.positions[prevPos]), + .errPos = state.positions[pos] }); } -static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) +static void dupAttr(const EvalState & state, Symbol attr, const PosIdx pos, const PosIdx prevPos) { throw ParseError({ - .msg = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos), - .errPos = pos + .msg = hintfmt("attribute '%1%' already defined at %2%", state.symbols[attr], state.positions[prevPos]), + .errPos = state.positions[pos] }); } -static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, - Expr * e, const Pos & pos) +static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, + Expr * e, const PosIdx pos, const nix::EvalState & state) { AttrPath::iterator i; // All attrpaths have at least one attr @@ -104,15 +99,15 @@ // Checking attrPath validity. // =========================== for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { - if (i->symbol.set()) { + if (i->symbol) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { if (!j->second.inherited) { ExprAttrs * attrs2 = dynamic_cast(j->second.e); - if (!attrs2) dupAttr(attrPath, pos, j->second.pos); + if (!attrs2) dupAttr(state, attrPath, pos, j->second.pos); attrs = attrs2; } else - dupAttr(attrPath, pos, j->second.pos); + dupAttr(state, attrPath, pos, j->second.pos); } else { ExprAttrs * nested = new ExprAttrs; attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); @@ -126,7 +121,7 @@ } // Expr insertion. // ========================== - if (i->symbol.set()) { + if (i->symbol) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { // This attr path is already defined. However, if both @@ -139,11 +134,11 @@ for (auto & ad : ae->attrs) { auto j2 = jAttrs->attrs.find(ad.first); if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. - dupAttr(ad.first, j2->second.pos, ad.second.pos); + dupAttr(state, ad.first, j2->second.pos, ad.second.pos); jAttrs->attrs.emplace(ad.first, ad.second); } } else { - dupAttr(attrPath, pos, j->second.pos); + dupAttr(state, attrPath, pos, j->second.pos); } } else { // This attr path is not defined. Let's create it. @@ -157,14 +152,14 @@ static Formals * toFormals(ParseData & data, ParserFormals * formals, - Pos pos = noPos, Symbol arg = {}) + PosIdx pos = noPos, Symbol arg = {}) { std::sort(formals->formals.begin(), formals->formals.end(), [] (const auto & a, const auto & b) { return std::tie(a.name, a.pos) < std::tie(b.name, b.pos); }); - std::optional> duplicate; + std::optional> duplicate; for (size_t i = 0; i + 1 < formals->formals.size(); i++) { if (formals->formals[i].name != formals->formals[i + 1].name) continue; @@ -173,18 +168,18 @@ } if (duplicate) throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", duplicate->first), - .errPos = duplicate->second + .msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[duplicate->first]), + .errPos = data.state.positions[duplicate->second] }); Formals result; result.ellipsis = formals->ellipsis; result.formals = std::move(formals->formals); - if (arg.set() && result.has(arg)) + if (arg && result.has(arg)) throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", arg), - .errPos = pos + .msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[arg]), + .errPos = data.state.positions[pos] }); delete formals; @@ -192,8 +187,8 @@ } -static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, - std::vector > > & es) +static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, + std::vector>> && es) { if (es.empty()) return new ExprString(""); @@ -233,7 +228,7 @@ } /* Strip spaces from each line. */ - std::vector > * es2 = new std::vector >; + auto * es2 = new std::vector>; atStartOfLine = true; size_t curDropped = 0; size_t n = es.size(); @@ -273,7 +268,7 @@ s2 = std::string(s2, 0, p + 1); } - es2->emplace_back(i->first, new ExprString(s2)); + es2->emplace_back(i->first, new ExprString(std::move(s2))); }; for (; i != es.end(); ++i, --n) { std::visit(overloaded { trimExpr, trimString }, i->second); @@ -284,9 +279,9 @@ } -static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data) +static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) { - return Pos(data->origin, data->file, loc.first_line, loc.first_column); + return data->state.positions.add(data->origin, loc.first_line, loc.first_column); } #define CUR_POS makeCurPos(*yylocp, data) @@ -299,7 +294,7 @@ { data->error = { .msg = hintfmt(error), - .errPos = makeCurPos(*loc, data) + .errPos = data->state.positions[makeCurPos(*loc, data)] }; } @@ -320,8 +315,8 @@ StringToken uri; StringToken str; std::vector * attrNames; - std::vector > * string_parts; - std::vector > > * ind_string_parts; + std::vector> * string_parts; + std::vector>> * ind_string_parts; } %type start expr expr_function expr_if expr_op @@ -369,15 +364,15 @@ : ID ':' expr_function { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); } | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), toFormals(*data, $2), $5); } + { $$ = new ExprLambda(CUR_POS, toFormals(*data, $2), $5); } | '{' formals '}' '@' ID ':' expr_function { - Symbol arg = data->symbols.create($5); + auto arg = data->symbols.create($5); $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $2, CUR_POS, arg), $7); } | ID '@' '{' formals '}' ':' expr_function { - Symbol arg = data->symbols.create($1); + auto arg = data->symbols.create($1); $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $4, CUR_POS, arg), $7); } | ASSERT expr ';' expr_function @@ -388,7 +383,7 @@ { if (!$2->dynamicAttrs.empty()) throw ParseError({ .msg = hintfmt("dynamic attributes not allowed in let"), - .errPos = CUR_POS + .errPos = data->state.positions[CUR_POS] }); $$ = new ExprLet($2, $4); } @@ -405,21 +400,21 @@ | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } - | expr_op '>' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } - | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); } - | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); } - | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); } - | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); } - | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } + | expr_op '<' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } + | expr_op '>' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } + | expr_op AND expr_op { $$ = new ExprOpAnd(makeCurPos(@2, data), $1, $3); } + | expr_op OR expr_op { $$ = new ExprOpOr(makeCurPos(@2, data), $1, $3); } + | expr_op IMPL expr_op { $$ = new ExprOpImpl(makeCurPos(@2, data), $1, $3); } + | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(makeCurPos(@2, data), $1, $3); } + | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; } | expr_op '+' expr_op - { $$ = new ExprConcatStrings(CUR_POS, false, new std::vector >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); } - | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); } - | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); } - | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); } - | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } + { $$ = new ExprConcatStrings(makeCurPos(@2, data), false, new std::vector >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); } + | expr_op '-' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__sub")), {$1, $3}); } + | expr_op '*' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__mul")), {$1, $3}); } + | expr_op '/' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__div")), {$1, $3}); } + | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(makeCurPos(@2, data), $1, $3); } | expr_app ; @@ -436,14 +431,14 @@ expr_select : expr_simple '.' attrpath - { $$ = new ExprSelect(CUR_POS, $1, *$3, 0); } + { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), nullptr); delete $3; } | expr_simple '.' attrpath OR_KW expr_select - { $$ = new ExprSelect(CUR_POS, $1, *$3, $5); } + { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; } | /* Backwards compatibility: because Nixpkgs has a rarely used function named ‘or’, allow stuff like ‘map or [...]’. */ expr_simple OR_KW { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); } - | expr_simple { $$ = $1; } + | expr_simple ; expr_simple @@ -458,9 +453,10 @@ | FLOAT { $$ = new ExprFloat($1); } | '"' string_parts '"' { $$ = $2; } | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { - $$ = stripIndentation(CUR_POS, data->symbols, *$2); + $$ = stripIndentation(CUR_POS, data->symbols, std::move(*$2)); + delete $2; } - | path_start PATH_END { $$ = $1; } + | path_start PATH_END | path_start string_parts_interpolated PATH_END { $2->insert($2->begin(), {makeCurPos(@1, data), $1}); $$ = new ExprConcatStrings(CUR_POS, false, $2); @@ -470,14 +466,14 @@ $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__findFile")), {new ExprVar(data->symbols.create("__nixPath")), - new ExprString(path)}); + new ExprString(std::move(path))}); } | URI { - static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals); + static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals); if (noURLLiterals) throw ParseError({ .msg = hintfmt("URL literals are disabled"), - .errPos = CUR_POS + .errPos = data->state.positions[CUR_POS] }); $$ = new ExprString(std::string($1)); } @@ -503,9 +499,9 @@ : string_parts_interpolated STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), new ExprString(std::string($2))); } | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); } - | DOLLAR_CURLY expr '}' { $$ = new std::vector >; $$->emplace_back(makeCurPos(@1, data), $2); } + | DOLLAR_CURLY expr '}' { $$ = new std::vector>; $$->emplace_back(makeCurPos(@1, data), $2); } | STR DOLLAR_CURLY expr '}' { - $$ = new std::vector >; + $$ = new std::vector>; $$->emplace_back(makeCurPos(@1, data), new ExprString(std::string($1))); $$->emplace_back(makeCurPos(@2, data), $3); } @@ -513,13 +509,19 @@ path_start : PATH { - Path path(absPath({$1.p, $1.l}, data->basePath)); + Path path(absPath({$1.p, $1.l}, data->basePath.path.abs())); /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; $$ = new ExprPath(path); } | HPATH { + if (evalSettings.pureEval) { + throw Error( + "the path '%s' can not be resolved in pure mode", + std::string_view($1.p, $1.l) + ); + } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); $$ = new ExprPath(path); } @@ -528,28 +530,30 @@ ind_string_parts : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); } | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); } - | { $$ = new std::vector > >; } + | { $$ = new std::vector>>; } ; binds - : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data)); } + : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, std::move(*$2), $4, makeCurPos(@2, data), data->state); delete $2; } | binds INHERIT attrs ';' { $$ = $1; for (auto & i : *$3) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) - dupAttr(i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos); - Pos pos = makeCurPos(@3, data); + dupAttr(data->state, i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos); + auto pos = makeCurPos(@3, data); $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); } + delete $3; } | binds INHERIT '(' expr ')' attrs ';' { $$ = $1; /* !!! Should ensure sharing of the expression in $4. */ for (auto & i : *$6) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) - dupAttr(i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); + dupAttr(data->state, i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data))); } + delete $6; } | { $$ = new ExprAttrs(makeCurPos(@0, data)); } ; @@ -565,7 +569,7 @@ } else throw ParseError({ .msg = hintfmt("dynamic attributes not allowed in inherit"), - .errPos = makeCurPos(@2, data) + .errPos = data->state.positions[makeCurPos(@2, data)] }); } | { $$ = new AttrPath; } @@ -595,7 +599,7 @@ ; attr - : ID { $$ = $1; } + : ID | OR_KW { $$ = {"or", 2}; } ; @@ -611,9 +615,9 @@ formals : formal ',' formals - { $$ = $3; $$->formals.push_back(*$1); } + { $$ = $3; $$->formals.emplace_back(*$1); delete $1; } | formal - { $$ = new ParserFormals; $$->formals.push_back(*$1); $$->ellipsis = false; } + { $$ = new ParserFormals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; } | { $$ = new ParserFormals; $$->ellipsis = false; } | ELLIPSIS @@ -621,8 +625,8 @@ ; formal - : ID { $$ = new Formal(CUR_POS, data->symbols.create($1), 0); } - | ID '?' expr { $$ = new Formal(CUR_POS, data->symbols.create($1), $3); } + : ID { $$ = new Formal{CUR_POS, data->symbols.create($1), 0}; } + | ID '?' expr { $$ = new Formal{CUR_POS, data->symbols.create($1), $3}; } ; %% @@ -637,29 +641,26 @@ #include "filetransfer.hh" #include "fetchers.hh" #include "store-api.hh" +#include "flake/flake.hh" namespace nix { -Expr * EvalState::parse(char * text, size_t length, FileOrigin origin, - const PathView path, const PathView basePath, StaticEnv & staticEnv) +Expr * EvalState::parse( + char * text, + size_t length, + Pos::Origin origin, + const SourcePath & basePath, + std::shared_ptr & staticEnv) { yyscan_t scanner; - ParseData data(*this); - data.origin = origin; - switch (origin) { - case foFile: - data.file = data.symbols.create(path); - break; - case foStdin: - case foString: - data.file = data.symbols.create(text); - break; - default: - assert(false); - } - data.basePath = basePath; + ParseData data { + .state = *this, + .symbols = symbols, + .basePath = std::move(basePath), + .origin = {origin}, + }; yylex_init(&scanner); yy_scan_buffer(text, length, scanner); @@ -668,61 +669,50 @@ if (res) throw ParseError(data.error.value()); - data.result->bindVars(staticEnv); + data.result->bindVars(*this, staticEnv); return data.result; } -Path resolveExprPath(Path path) +SourcePath resolveExprPath(const SourcePath & path) { - assert(path[0] == '/'); - - unsigned int followCount = 0, maxFollow = 1024; - /* If `path' is a symlink, follow it. This is so that relative path references work. */ - struct stat st; - while (true) { - // Basic cycle/depth limit to avoid infinite loops. - if (++followCount >= maxFollow) - throw Error("too many symbolic links encountered while traversing the path '%s'", path); - st = lstat(path); - if (!S_ISLNK(st.st_mode)) break; - path = absPath(readLink(path), dirOf(path)); - } + auto path2 = path.resolveSymlinks(); /* If `path' refers to a directory, append `/default.nix'. */ - if (S_ISDIR(st.st_mode)) - path = canonPath(path + "/default.nix"); + if (path2.lstat().type == InputAccessor::tDirectory) + return path2 + "default.nix"; - return path; + return path2; } -Expr * EvalState::parseExprFromFile(const Path & path) +Expr * EvalState::parseExprFromFile(const SourcePath & path) { return parseExprFromFile(path, staticBaseEnv); } -Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv) +Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) { - auto buffer = readFile(path); - // readFile should have left some extra space for terminators + auto buffer = path.readFile(); + // readFile hopefully have left some extra space for terminators buffer.append("\0\0", 2); - return parse(buffer.data(), buffer.size(), foFile, path, dirOf(path), staticEnv); + return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv); } -Expr * EvalState::parseExprFromString(std::string s, const Path & basePath, StaticEnv & staticEnv) +Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr & staticEnv) { - s.append("\0\0", 2); - return parse(s.data(), s.size(), foString, "", basePath, staticEnv); + auto s = make_ref(std::move(s_)); + s->append("\0\0", 2); + return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv); } -Expr * EvalState::parseExprFromString(std::string s, const Path & basePath) +Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath) { return parseExprFromString(std::move(s), basePath, staticBaseEnv); } @@ -730,11 +720,12 @@ Expr * EvalState::parseStdin() { - //Activity act(*logger, lvlTalkative, format("parsing standard input")); + //Activity act(*logger, lvlTalkative, "parsing standard input"); auto buffer = drainFD(0); // drainFD should have left some extra space for terminators buffer.append("\0\0", 2); - return parse(buffer.data(), buffer.size(), foStdin, "", absPath("."), staticBaseEnv); + auto s = make_ref(std::move(buffer)); + return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv); } @@ -754,13 +745,13 @@ } -Path EvalState::findFile(const std::string_view path) +SourcePath EvalState::findFile(const std::string_view path) { return findFile(searchPath, path); } -Path EvalState::findFile(SearchPath & searchPath, const std::string_view path, const Pos & pos) +SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos) { for (auto & i : searchPath) { std::string suffix; @@ -776,19 +767,19 @@ auto r = resolveSearchPathElem(i); if (!r.first) continue; Path res = r.second + suffix; - if (pathExists(res)) return canonPath(res); + if (pathExists(res)) return CanonPath(canonPath(res)); } if (hasPrefix(path, "nix/")) - return concatStrings(corepkgsPrefix, path.substr(4)); + return CanonPath(concatStrings(corepkgsPrefix, path.substr(4))); - throw ThrownError({ + debugThrow(ThrownError({ .msg = hintfmt(evalSettings.pureEval ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", path), - .errPos = pos - }); + .errPos = positions[pos] + }), 0, 0); } @@ -799,17 +790,28 @@ std::pair res; - if (isUri(elem.second)) { + if (EvalSettings::isPseudoUrl(elem.second)) { try { - res = { true, store->toRealPath(fetchers::downloadTarball( - store, resolveUri(elem.second), "source", false).first.storePath) }; + auto storePath = fetchers::downloadTarball( + store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).first.storePath; + res = { true, store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) }); res = { false, "" }; } - } else { + } + + else if (hasPrefix(elem.second, "flake:")) { + experimentalFeatureSettings.require(Xp::Flakes); + auto flakeRef = parseFlakeRef(elem.second.substr(6), {}, true, false); + debug("fetching flake search path element '%s''", elem.second); + auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; + res = { true, store->toRealPath(storePath) }; + } + + else { auto path = absPath(elem.second); if (pathExists(path)) res = { true, path }; @@ -821,7 +823,7 @@ } } - debug(format("resolved search path element '%s' to '%s'") % elem.second % res.second); + debug("resolved search path element '%s' to '%s'", elem.second, res.second); searchPathResolved[elem.second] = res; return res; diff -Nru nix-2.8.0/src/libexpr/paths.cc nix-2.16.1+dfsg/src/libexpr/paths.cc --- nix-2.8.0/src/libexpr/paths.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/paths.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,10 @@ +#include "eval.hh" + +namespace nix { + +SourcePath EvalState::rootPath(CanonPath path) +{ + return std::move(path); +} + +} diff -Nru nix-2.8.0/src/libexpr/primops/context.cc nix-2.16.1+dfsg/src/libexpr/primops/context.cc --- nix-2.8.0/src/libexpr/primops/context.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/primops/context.cc 2023-06-06 10:55:23.000000000 +0000 @@ -5,20 +5,20 @@ namespace nix { -static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto s = state.coerceToString(pos, *args[0], context); + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext"); v.mkString(*s); } static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); -static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - state.forceString(*args[0], context, pos); + NixStringContext context; + state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.hasContext"); v.mkBool(!context.empty()); } @@ -31,14 +31,22 @@ source-only deployment). This primop marks the string context so that builtins.derivation adds the path to drv.inputSrcs rather than drv.inputDrvs. */ -static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto s = state.coerceToString(pos, *args[0], context); + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency"); - PathSet context2; - for (auto & p : context) - context2.insert(p.at(0) == '=' ? std::string(p, 1) : p); + NixStringContext context2; + for (auto && c : context) { + if (auto * ptr = std::get_if(&c)) { + context2.emplace(NixStringContextElem::Opaque { + .path = ptr->drvPath + }); + } else { + /* Can reuse original item */ + context2.emplace(std::move(c)); + } + } v.mkString(*s, context2); } @@ -65,43 +73,28 @@ Note that for a given path any combination of the above attributes may be present. */ -static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { struct ContextInfo { bool path = false; bool allOutputs = false; Strings outputs; }; - PathSet context; - state.forceString(*args[0], context, pos); - auto contextInfos = std::map(); - for (const auto & p : context) { - Path drv; - std::string output; - const Path * path = &p; - if (p.at(0) == '=') { - drv = std::string(p, 1); - path = &drv; - } else if (p.at(0) == '!') { - NixStringContextElem ctx = decodeContext(*state.store, p); - drv = state.store->printStorePath(ctx.first); - output = ctx.second; - path = &drv; - } - auto isPath = drv.empty(); - auto isAllOutputs = (!drv.empty()) && output.empty(); - - auto iter = contextInfos.find(*path); - if (iter == contextInfos.end()) { - contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}}); - } else { - if (isPath) - iter->second.path = true; - else if (isAllOutputs) - iter->second.allOutputs = true; - else - iter->second.outputs.emplace_back(std::move(output)); - } + NixStringContext context; + state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext"); + auto contextInfos = std::map(); + for (auto && i : context) { + std::visit(overloaded { + [&](NixStringContextElem::DrvDeep && d) { + contextInfos[std::move(d.drvPath)].allOutputs = true; + }, + [&](NixStringContextElem::Built && b) { + contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output)); + }, + [&](NixStringContextElem::Opaque && o) { + contextInfos[std::move(o.path)].path = true; + }, + }, ((NixStringContextElem &&) i).raw()); } auto attrs = state.buildBindings(contextInfos.size()); @@ -120,7 +113,7 @@ for (const auto & [i, output] : enumerate(info.second.outputs)) (outputsVal.listElems()[i] = state.allocValue())->mkString(output); } - attrs.alloc(info.first).mkAttrs(infoAttrs); + attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs); } v.mkAttrs(attrs); @@ -134,55 +127,64 @@ See the commentary above unsafeGetContext for details of the context representation. */ -static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto orig = state.forceString(*args[0], context, pos); + NixStringContext context; + auto orig = state.forceString(*args[0], context, noPos, "while evaluating the first argument passed to builtins.appendContext"); - state.forceAttrs(*args[1], pos); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.appendContext"); auto sPath = state.symbols.create("path"); auto sAllOutputs = state.symbols.create("allOutputs"); for (auto & i : *args[1]->attrs) { - if (!state.store->isStorePath(i.name)) + const auto & name = state.symbols[i.name]; + if (!state.store->isStorePath(name)) throw EvalError({ - .msg = hintfmt("Context key '%s' is not a store path", i.name), - .errPos = *i.pos + .msg = hintfmt("context key '%s' is not a store path", name), + .errPos = state.positions[i.pos] }); + auto namePath = state.store->parseStorePath(name); if (!settings.readOnlyMode) - state.store->ensurePath(state.store->parseStorePath(i.name)); - state.forceAttrs(*i.value, *i.pos); + state.store->ensurePath(namePath); + state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); auto iter = i.value->attrs->find(sPath); if (iter != i.value->attrs->end()) { - if (state.forceBool(*iter->value, *iter->pos)) - context.insert(i.name); + if (state.forceBool(*iter->value, iter->pos, "while evaluating the `path` attribute of a string context")) + context.emplace(NixStringContextElem::Opaque { + .path = namePath, + }); } iter = i.value->attrs->find(sAllOutputs); if (iter != i.value->attrs->end()) { - if (state.forceBool(*iter->value, *iter->pos)) { - if (!isDerivation(i.name)) { + if (state.forceBool(*iter->value, iter->pos, "while evaluating the `allOutputs` attribute of a string context")) { + if (!isDerivation(name)) { throw EvalError({ - .msg = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), - .errPos = *i.pos + .msg = hintfmt("tried to add all-outputs context of %s, which is not a derivation, to a string", name), + .errPos = state.positions[i.pos] }); } - context.insert("=" + std::string(i.name)); + context.emplace(NixStringContextElem::DrvDeep { + .drvPath = namePath, + }); } } iter = i.value->attrs->find(state.sOutputs); if (iter != i.value->attrs->end()) { - state.forceList(*iter->value, *iter->pos); - if (iter->value->listSize() && !isDerivation(i.name)) { + state.forceList(*iter->value, iter->pos, "while evaluating the `outputs` attribute of a string context"); + if (iter->value->listSize() && !isDerivation(name)) { throw EvalError({ - .msg = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name), - .errPos = *i.pos + .msg = hintfmt("tried to add derivation output context of %s, which is not a derivation, to a string", name), + .errPos = state.positions[i.pos] }); } for (auto elem : iter->value->listItems()) { - auto name = state.forceStringNoCtx(*elem, *iter->pos); - context.insert(concatStrings("!", name, "!", i.name)); + auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); + context.emplace(NixStringContextElem::Built { + .drvPath = namePath, + .output = std::string { outputName }, + }); } } } diff -Nru nix-2.8.0/src/libexpr/primops/fetchClosure.cc nix-2.16.1+dfsg/src/libexpr/primops/fetchClosure.cc --- nix-2.8.0/src/libexpr/primops/fetchClosure.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/primops/fetchClosure.cc 2023-06-06 10:55:23.000000000 +0000 @@ -5,9 +5,9 @@ namespace nix { -static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchClosure"); std::optional fromStoreUrl; std::optional fromPath; @@ -15,40 +15,45 @@ std::optional toPath; for (auto & attr : *args[0]->attrs) { - if (attr.name == "fromPath") { - PathSet context; - fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context); + const auto & attrName = state.symbols[attr.name]; + + if (attrName == "fromPath") { + NixStringContext context; + fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, + "while evaluating the 'fromPath' attribute passed to builtins.fetchClosure"); } - else if (attr.name == "toPath") { - state.forceValue(*attr.value, *attr.pos); + else if (attrName == "toPath") { + state.forceValue(*attr.value, attr.pos); toCA = true; if (attr.value->type() != nString || attr.value->string.s != std::string("")) { - PathSet context; - toPath = state.coerceToStorePath(*attr.pos, *attr.value, context); + NixStringContext context; + toPath = state.coerceToStorePath(attr.pos, *attr.value, context, + "while evaluating the 'toPath' attribute passed to builtins.fetchClosure"); } } - else if (attr.name == "fromStore") - fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (attrName == "fromStore") + fromStoreUrl = state.forceStringNoCtx(*attr.value, attr.pos, + "while evaluating the 'fromStore' attribute passed to builtins.fetchClosure"); else throw Error({ - .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attr.name), - .errPos = pos + .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attrName), + .errPos = state.positions[pos] }); } if (!fromPath) throw Error({ .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"), - .errPos = pos + .errPos = state.positions[pos] }); if (!fromStoreUrl) throw Error({ .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), - .errPos = pos + .errPos = state.positions[pos] }); auto parsedURL = parseURL(*fromStoreUrl); @@ -58,13 +63,13 @@ !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file")) throw Error({ .msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"), - .errPos = pos + .errPos = state.positions[pos] }); if (!parsedURL.query.empty()) throw Error({ .msg = hintfmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl), - .errPos = pos + .errPos = state.positions[pos] }); auto fromStore = openStore(parsedURL.to_string()); @@ -80,7 +85,7 @@ state.store->printStorePath(*fromPath), state.store->printStorePath(i->second), state.store->printStorePath(*toPath)), - .errPos = pos + .errPos = state.positions[pos] }); if (!toPath) throw Error({ @@ -89,7 +94,7 @@ "please set this in the 'toPath' attribute passed to 'fetchClosure'", state.store->printStorePath(*fromPath), state.store->printStorePath(i->second)), - .errPos = pos + .errPos = state.positions[pos] }); } } else { @@ -105,12 +110,11 @@ throw Error({ .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't", state.store->printStorePath(*toPath)), - .errPos = pos + .errPos = state.positions[pos] }); } - auto toPathS = state.store->printStorePath(*toPath); - v.mkString(toPathS, {toPathS}); + state.mkStorePathString(*toPath, v); } static RegisterPrimOp primop_fetchClosure({ diff -Nru nix-2.8.0/src/libexpr/primops/fetchMercurial.cc nix-2.16.1+dfsg/src/libexpr/primops/fetchMercurial.cc --- nix-2.8.0/src/libexpr/primops/fetchMercurial.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/primops/fetchMercurial.cc 2023-06-06 10:55:23.000000000 +0000 @@ -7,50 +7,52 @@ namespace nix { -static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::string url; std::optional rev; std::optional ref; std::string_view name = "source"; - PathSet context; + NixStringContext context; state.forceValue(*args[0], pos); if (args[0]->type() == nAttrs) { - state.forceAttrs(*args[0], pos); - for (auto & attr : *args[0]->attrs) { - std::string_view n(attr.name); + std::string_view n(state.symbols[attr.name]); if (n == "url") - url = state.coerceToString(*attr.pos, *attr.value, context, false, false).toOwned(); + url = state.coerceToString(attr.pos, *attr.value, context, + "while evaluating the `url` attribute passed to builtins.fetchMercurial", + false, false).toOwned(); else if (n == "rev") { // Ugly: unlike fetchGit, here the "rev" attribute can // be both a revision or a branch/tag name. - auto value = state.forceStringNoCtx(*attr.value, *attr.pos); + auto value = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `rev` attribute passed to builtins.fetchMercurial"); if (std::regex_match(value.begin(), value.end(), revRegex)) rev = Hash::parseAny(value, htSHA1); else ref = value; } else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); + name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `name` attribute passed to builtins.fetchMercurial"); else throw EvalError({ - .msg = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), - .errPos = *attr.pos + .msg = hintfmt("unsupported argument '%s' to 'fetchMercurial'", state.symbols[attr.name]), + .errPos = state.positions[attr.pos] }); } if (url.empty()) throw EvalError({ .msg = hintfmt("'url' argument required"), - .errPos = pos + .errPos = state.positions[pos] }); } else - url = state.coerceToString(pos, *args[0], context, false, false).toOwned(); + url = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to builtins.fetchMercurial", + false, false).toOwned(); // FIXME: git externals probably can be used to bypass the URI // whitelist. Ah well. @@ -71,8 +73,7 @@ auto [tree, input2] = input.fetch(state.store); auto attrs2 = state.buildBindings(8); - auto storePath = state.store->printStorePath(tree.storePath); - attrs2.alloc(state.sOutPath).mkString(storePath, {storePath}); + state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to diff -Nru nix-2.8.0/src/libexpr/primops/fetchTree.cc nix-2.16.1+dfsg/src/libexpr/primops/fetchTree.cc --- nix-2.8.0/src/libexpr/primops/fetchTree.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/primops/fetchTree.cc 2023-06-06 10:55:23.000000000 +0000 @@ -4,6 +4,7 @@ #include "fetchers.hh" #include "filetransfer.hh" #include "registry.hh" +#include "url.hh" #include #include @@ -23,9 +24,8 @@ auto attrs = state.buildBindings(8); - auto storePath = state.store->printStorePath(tree.storePath); - attrs.alloc(state.sOutPath).mkString(storePath, {storePath}); + state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. @@ -68,7 +68,16 @@ std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") { state.checkURI(uri); - return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri; + if (uri.find("://") == std::string::npos) { + const auto p = ParsedURL { + .scheme = defaultScheme, + .authority = "", + .path = uri + }; + return p.to_string(); + } else { + return uri; + } } std::string fixURIForGit(std::string uri, EvalState & state) @@ -90,68 +99,70 @@ static void fetchTree( EvalState & state, - const Pos & pos, + const PosIdx pos, Value * * args, Value & v, std::optional type, const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; - PathSet context; + NixStringContext context; state.forceValue(*args[0], pos); if (args[0]->type() == nAttrs) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchTree"); fetchers::Attrs attrs; if (auto aType = args[0]->attrs->get(state.sType)) { if (type) - throw Error({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("unexpected attribute 'type'"), - .errPos = pos - }); - type = state.forceStringNoCtx(*aType->value, *aType->pos); + .errPos = state.positions[pos] + })); + type = state.forceStringNoCtx(*aType->value, aType->pos, "while evaluating the `type` attribute passed to builtins.fetchTree"); } else if (!type) - throw Error({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), - .errPos = pos - }); + .errPos = state.positions[pos] + })); attrs.emplace("type", type.value()); for (auto & attr : *args[0]->attrs) { if (attr.name == state.sType) continue; - state.forceValue(*attr.value, *attr.pos); + state.forceValue(*attr.value, attr.pos); if (attr.value->type() == nPath || attr.value->type() == nString) { - auto s = state.coerceToString(*attr.pos, *attr.value, context, false, false).toOwned(); - attrs.emplace(attr.name, - attr.name == "url" + auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned(); + attrs.emplace(state.symbols[attr.name], + state.symbols[attr.name] == "url" ? type == "git" ? fixURIForGit(s, state) : fixURI(s, state) : s); } else if (attr.value->type() == nBool) - attrs.emplace(attr.name, Explicit{attr.value->boolean}); + attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean}); else if (attr.value->type() == nInt) - attrs.emplace(attr.name, uint64_t(attr.value->integer)); + attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer)); else - throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", - attr.name, showType(*attr.value)); + state.debugThrowLastTrace(TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", + state.symbols[attr.name], showType(*attr.value))); } if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) - throw Error({ - .msg = hintfmt("attribute 'name' isn't supported in call to 'fetchTree'"), - .errPos = pos - }); + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"), + .errPos = state.positions[pos] + })); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - auto url = state.coerceToString(pos, *args[0], context, false, false).toOwned(); + auto url = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to the fetcher", + false, false).toOwned(); if (type == "git") { fetchers::Attrs attrs; @@ -167,7 +178,7 @@ input = lookupInRegistries(state.store, input).first; if (evalSettings.pureEval && !input.isLocked()) - throw Error("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", pos); + state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos])); auto [tree, input2] = input.fetch(state.store); @@ -176,16 +187,16 @@ emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); } -static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); } // FIXME: document static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); -static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, +static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, const std::string & who, bool unpack, std::string name) { std::optional url; @@ -195,32 +206,31 @@ if (args[0]->type() == nAttrs) { - state.forceAttrs(*args[0], pos); - for (auto & attr : *args[0]->attrs) { - std::string n(attr.name); + std::string_view n(state.symbols[attr.name]); if (n == "url") - url = state.forceStringNoCtx(*attr.value, *attr.pos); + url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch"); else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), htSHA256); else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); + name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch"); else - throw EvalError({ - .msg = hintfmt("unsupported argument '%s' to '%s'", attr.name, who), - .errPos = *attr.pos - }); - } + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("unsupported argument '%s' to '%s'", n, who), + .errPos = state.positions[attr.pos] + })); + } if (!url) - throw EvalError({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("'url' argument required"), - .errPos = pos - }); + .errPos = state.positions[pos] + })); } else - url = state.forceStringNoCtx(*args[0], pos); + url = state.forceStringNoCtx(*args[0], pos, "while evaluating the url we should fetch"); - url = resolveUri(*url); + if (who == "fetchTarball") + url = evalSettings.resolvePseudoUrl(*url); state.checkURI(*url); @@ -228,14 +238,19 @@ name = baseNameOf(*url); if (evalSettings.pureEval && !expectedHash) - throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who)); // early exit if pinned and already in the store if (expectedHash && expectedHash->type == htSHA256) { - auto expectedPath = - unpack - ? state.store->makeFixedOutputPath(FileIngestionMethod::Recursive, *expectedHash, name, {}) - : state.store->makeFixedOutputPath(FileIngestionMethod::Flat, *expectedHash, name, {}); + auto expectedPath = state.store->makeFixedOutputPath( + name, + FixedOutputInfo { + .hash = { + .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .hash = *expectedHash, + }, + .references = {} + }); if (state.store->isValidPath(expectedPath)) { state.allowAndSetStorePathString(expectedPath, v); @@ -255,14 +270,14 @@ ? state.store->queryPathInfo(storePath)->narHash : hashFile(htSHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) - throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); + state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", + *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true))); } state.allowAndSetStorePathString(storePath, v); } -static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_fetchurl(EvalState & state, const PosIdx pos, Value * * args, Value & v) { fetch(state, pos, args, v, "fetchurl", false, ""); } @@ -271,14 +286,14 @@ .name = "__fetchurl", .args = {"url"}, .doc = R"( - Download the specified URL and return the path of the downloaded - file. This function is not available if [restricted evaluation - mode](../command-ref/conf-file.md) is enabled. + Download the specified URL and return the path of the downloaded file. + + Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", .fun = prim_fetchurl, }); -static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_fetchTarball(EvalState & state, const PosIdx pos, Value * * args, Value & v) { fetch(state, pos, args, v, "fetchTarball", true, "source"); } @@ -323,13 +338,12 @@ stdenv.mkDerivation { … } ``` - This function is not available if [restricted evaluation - mode](../command-ref/conf-file.md) is enabled. + Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", .fun = prim_fetchTarball, }); -static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_fetchGit(EvalState & state, const PosIdx pos, Value * * args, Value & v) { fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); } @@ -342,32 +356,44 @@ of the repo at that URL is fetched. Otherwise, it can be an attribute with the following attributes (all except `url` optional): - - url\ - The URL of the repo. + - `url` + + The URL of the repo. + + - `name` (default: *basename of the URL*) + + The name of the directory the repo should be exported to in the store. + + - `rev` (default: *the tip of `ref`*) + + The [Git revision] to fetch. + This is typically a commit hash. + + [Git revision]: https://git-scm.com/docs/git-rev-parse#_specifying_revisions + + - `ref` (default: `HEAD`) + + The [Git reference] under which to look for the requested revision. + This is often a branch or tag name. - - name\ - The name of the directory the repo should be exported to in the - store. Defaults to the basename of the URL. - - - rev\ - The git revision to fetch. Defaults to the tip of `ref`. - - - ref\ - The git ref to look for the requested revision under. This is - often a branch or tag name. Defaults to `HEAD`. - - By default, the `ref` value is prefixed with `refs/heads/`. As - of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts - with `refs/`. - - - submodules\ - A Boolean parameter that specifies whether submodules should be - checked out. Defaults to `false`. - - - allRefs\ - Whether to fetch all refs of the repository. With this argument being - true, it's possible to load a `rev` from *any* `ref` (by default only - `rev`s from the specified `ref` are supported). + [Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References + + By default, the `ref` value is prefixed with `refs/heads/`. + As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/`. + + - `submodules` (default: `false`) + + A Boolean parameter that specifies whether submodules should be checked out. + + - `shallow` (default: `false`) + + A Boolean parameter that specifies whether fetching a shallow clone is allowed. + + - `allRefs` + + Whether to fetch all references of the repository. + With this argument being true, it's possible to load a `rev` from *any* `ref` + (by default only `rev`s from the specified `ref` are supported). Here are some examples of how to use `fetchGit`. @@ -443,14 +469,20 @@ } ``` - > **Note** - > - > Nix will refetch the branch in accordance with - > the option `tarball-ttl`. + Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting. - > **Note** - > - > This behavior is disabled in *Pure evaluation mode*. + This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + + - To fetch the content of a checked-out work directory: + + ```nix + builtins.fetchGit ./work-dir + ``` + + If the URL points to a local directory, and no `ref` or `rev` is + given, `fetchGit` will use the current content of the checked-out + files, even if they are not committed or added to Git's index. It will + only consider files added to the Git repository, as listed by `git ls-files`. )", .fun = prim_fetchGit, }); diff -Nru nix-2.8.0/src/libexpr/primops/fromTOML.cc nix-2.16.1+dfsg/src/libexpr/primops/fromTOML.cc --- nix-2.8.0/src/libexpr/primops/fromTOML.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/primops/fromTOML.cc 2023-06-06 10:55:23.000000000 +0000 @@ -5,9 +5,9 @@ namespace nix { -static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & val) +static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val) { - auto toml = state.forceStringNoCtx(*args[0], pos); + auto toml = state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.fromTOML"); std::istringstream tomlStream(std::string{toml}); @@ -73,7 +73,7 @@ } catch (std::exception & e) { // TODO: toml::syntax_error throw EvalError({ .msg = hintfmt("while parsing a TOML string: %s", e.what()), - .errPos = pos + .errPos = state.positions[pos] }); } } diff -Nru nix-2.8.0/src/libexpr/primops.cc nix-2.16.1+dfsg/src/libexpr/primops.cc --- nix-2.8.0/src/libexpr/primops.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/primops.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,18 +1,20 @@ #include "archive.hh" #include "derivations.hh" +#include "downstream-placeholder.hh" #include "eval-inline.hh" #include "eval.hh" #include "globals.hh" #include "json-to-value.hh" #include "names.hh" +#include "references.hh" #include "store-api.hh" #include "util.hh" -#include "json.hh" #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" #include +#include #include #include @@ -37,29 +39,44 @@ InvalidPathError::InvalidPathError(const Path & path) : EvalError("path '%s' is not valid", path), path(path) {} -StringMap EvalState::realiseContext(const PathSet & context) +StringMap EvalState::realiseContext(const NixStringContext & context) { std::vector drvs; StringMap res; - for (auto & i : context) { - auto [ctx, outputName] = decodeContext(*store, i); - auto ctxS = store->printStorePath(ctx); - if (!store->isValidPath(ctx)) - throw InvalidPathError(store->printStorePath(ctx)); - if (!outputName.empty() && ctx.isDerivation()) { - drvs.push_back({ctx, {outputName}}); - } else { - res.insert_or_assign(ctxS, ctxS); - } + for (auto & c : context) { + auto ensureValid = [&](const StorePath & p) { + if (!store->isValidPath(p)) + debugThrowLastTrace(InvalidPathError(store->printStorePath(p))); + }; + std::visit(overloaded { + [&](const NixStringContextElem::Built & b) { + drvs.push_back(DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }); + ensureValid(b.drvPath); + }, + [&](const NixStringContextElem::Opaque & o) { + auto ctxS = store->printStorePath(o.path); + res.insert_or_assign(ctxS, ctxS); + ensureValid(o.path); + }, + [&](const NixStringContextElem::DrvDeep & d) { + /* Treat same as Opaque */ + auto ctxS = store->printStorePath(d.drvPath); + res.insert_or_assign(ctxS, ctxS); + ensureValid(d.drvPath); + }, + }, c.raw()); } if (drvs.empty()) return {}; if (!evalSettings.enableImportFromDerivation) - throw Error( + debugThrowLastTrace(Error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", - store->printStorePath(drvs.begin()->drvPath)); + store->printStorePath(drvs.begin()->drvPath))); /* Build/substitute the context. */ std::vector buildReqs; @@ -67,15 +84,12 @@ store->buildPaths(buildReqs); /* Get all the output paths corresponding to the placeholders we had */ - for (auto & [drvPath, outputs] : drvs) { - auto outputPaths = store->queryDerivationOutputMap(drvPath); - for (auto & outputName : outputs) { - if (outputPaths.count(outputName) == 0) - throw Error("derivation '%s' does not have an output named '%s'", - store->printStorePath(drvPath), outputName); + for (auto & drv : drvs) { + auto outputs = resolveDerivedPath(*store, drv); + for (auto & [outputName, outputPath] : outputs) { res.insert_or_assign( - downstreamPlaceholder(*store, drvPath, outputName), - store->printStorePath(outputPaths.at(outputName)) + DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), + store->printStorePath(outputPath) ); } } @@ -96,92 +110,82 @@ bool checkForPureEval = true; }; -static Path realisePath(EvalState & state, const Pos & pos, Value & v, const RealisePathFlags flags = {}) +static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {}) { - PathSet context; + NixStringContext context; - auto path = [&]() - { - try { - return state.coerceToPath(pos, v, context); - } catch (Error & e) { - e.addTrace(pos, "while realising the context of a path"); - throw; - } - }(); + auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); try { StringMap rewrites = state.realiseContext(context); - auto realPath = state.toRealPath(rewriteStrings(path, rewrites), context); + auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))); return flags.checkForPureEval ? state.checkSourcePath(realPath) : realPath; } catch (Error & e) { - e.addTrace(pos, "while realising the context of path '%s'", path); + e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); throw; } } -/* Add and attribute to the given attribute map from the output name to - the output path, or a placeholder. - - Where possible the path is used, but for floating CA derivations we - may not know it. For sake of determinism we always assume we don't - and instead put in a place holder. In either case, however, the - string context will contain the drv path and output name, so - downstream derivations will have the proper dependency, and in - addition, before building, the placeholder will be rewritten to be - the actual path. - - The 'drv' and 'drvPath' outputs must correspond. */ +/** + * Add and attribute to the given attribute map from the output name to + * the output path, or a placeholder. + * + * Where possible the path is used, but for floating CA derivations we + * may not know it. For sake of determinism we always assume we don't + * and instead put in a place holder. In either case, however, the + * string context will contain the drv path and output name, so + * downstream derivations will have the proper dependency, and in + * addition, before building, the placeholder will be rewritten to be + * the actual path. + * + * The 'drv' and 'drvPath' outputs must correspond. + */ static void mkOutputString( EvalState & state, BindingsBuilder & attrs, const StorePath & drvPath, - const BasicDerivation & drv, const std::pair & o) { - auto optOutputPath = o.second.path(*state.store, drv.name, o.first); - attrs.alloc(o.first).mkString( - optOutputPath - ? state.store->printStorePath(*optOutputPath) - /* Downstream we would substitute this for an actual path once - we build the floating CA derivation */ - /* FIXME: we need to depend on the basic derivation, not - derivation */ - : downstreamPlaceholder(*state.store, drvPath, o.first), - {"!" + o.first + "!" + state.store->printStorePath(drvPath)}); + state.mkOutputString( + attrs.alloc(o.first), + drvPath, + o.first, + o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first)); } /* Load and evaluate an expression from path specified by the argument. */ -static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vScope, Value & v) +static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v) { auto path = realisePath(state, pos, vPath); + auto path2 = path.path.abs(); // FIXME auto isValidDerivationInStore = [&]() -> std::optional { - if (!state.store->isStorePath(path)) + if (!state.store->isStorePath(path2)) return std::nullopt; - auto storePath = state.store->parseStorePath(path); - if (!(state.store->isValidPath(storePath) && isDerivation(path))) + auto storePath = state.store->parseStorePath(path2); + if (!(state.store->isValidPath(storePath) && isDerivation(path2))) return std::nullopt; return storePath; }; - if (auto optStorePath = isValidDerivationInStore()) { - auto storePath = *optStorePath; - Derivation drv = state.store->readDerivation(storePath); + if (auto storePath = isValidDerivationInStore()) { + Derivation drv = state.store->readDerivation(*storePath); auto attrs = state.buildBindings(3 + drv.outputs.size()); - attrs.alloc(state.sDrvPath).mkString(path, {"=" + path}); + attrs.alloc(state.sDrvPath).mkString(path2, { + NixStringContextElem::DrvDeep { .drvPath = *storePath }, + }); attrs.alloc(state.sName).mkString(drv.env["name"]); auto & outputsVal = attrs.alloc(state.sOutputs); state.mkList(outputsVal, drv.outputs.size()); for (const auto & [i, o] : enumerate(drv.outputs)) { - mkOutputString(state, attrs, storePath, drv, o); + mkOutputString(state, attrs, *storePath, o); (outputsVal.listElems()[i] = state.allocValue())->mkString(o.first); } @@ -192,34 +196,34 @@ state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , "/"), **state.vImportedDrvToDerivation); + , CanonPath::root), **state.vImportedDrvToDerivation); } - state.forceFunction(**state.vImportedDrvToDerivation, pos); + state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); v.mkApp(*state.vImportedDrvToDerivation, w); - state.forceAttrs(v, pos); + state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); } - else if (path == corepkgsPrefix + "fetchurl.nix") { + else if (path2 == corepkgsPrefix + "fetchurl.nix") { state.eval(state.parseExprFromString( #include "fetchurl.nix.gen.hh" - , "/"), v); + , CanonPath::root), v); } else { if (!vScope) state.evalFile(path, v); else { - state.forceAttrs(*vScope, pos); + state.forceAttrs(*vScope, pos, "while evaluating the first argument passed to builtins.scopedImport"); Env * env = &state.allocEnv(vScope->attrs->size()); env->up = &state.baseEnv; - StaticEnv staticEnv(false, &state.staticBaseEnv, vScope->attrs->size()); + auto staticEnv = std::make_shared(false, state.staticBaseEnv.get(), vScope->attrs->size()); unsigned int displ = 0; for (auto & attr : *vScope->attrs) { - staticEnv.vars.emplace_back(attr.name, displ); + staticEnv->vars.emplace_back(attr.name, displ); env->values[displ++] = attr.value; } @@ -237,7 +241,7 @@ static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info { .name = "scopedImport", .arity = 2, - .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) + .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { import(state, pos, *args[1], args[0], v); } @@ -246,10 +250,18 @@ static RegisterPrimOp primop_import({ .name = "import", .args = {"path"}, + // TODO turn "normal path values" into link below .doc = R"( - Load, parse and return the Nix expression in the file *path*. If - *path* is a directory, the file ` default.nix ` in that directory - is loaded. Evaluation aborts if the file doesn’t exist or contains + Load, parse and return the Nix expression in the file *path*. + + The value *path* can be a path, a string, or an attribute set with an + `__toString` attribute or a `outPath` attribute (as derivations or flake + inputs typically have). + + If *path* is a directory, the file `default.nix` in that directory + is loaded. + + Evaluation aborts if the file doesn’t exist or contains an incorrect Nix expression. `import` implements Nix’s module system: you can put any Nix expression (such as a set or a function) in a separate file, and use it from Nix expressions in @@ -259,7 +271,7 @@ > > Unlike some languages, `import` is a regular function in Nix. > Paths using the angle bracket syntax (e.g., `import` *\*) - > are [normal path values](language-values.md). + > are normal [path values](@docroot@/language/values.md#type-path). A Nix expression loaded by `import` must not contain any *free variables* (identifiers that are not defined in the Nix expression @@ -299,7 +311,7 @@ (The function argument doesn’t have to be called `x` in `foo.nix`; any name would work.) )", - .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) + .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { import(state, pos, *args[0], nullptr, v); } @@ -310,25 +322,24 @@ extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); /* Load a ValueInitializer from a DSO and return whatever it initializes */ -void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v) +void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto path = realisePath(state, pos, *args[0]); - std::string sym(state.forceStringNoCtx(*args[1], pos)); + std::string sym(state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.importNative")); - void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); + void *handle = dlopen(path.path.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) - throw EvalError("could not open '%1%': %2%", path, dlerror()); + state.debugThrowLastTrace(EvalError("could not open '%1%': %2%", path, dlerror())); dlerror(); ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str()); if(!func) { char *message = dlerror(); if (message) - throw EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message); + state.debugThrowLastTrace(EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message)); else - throw EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", - sym, path); + state.debugThrowLastTrace(EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", sym, path)); } (func)(state, v); @@ -338,52 +349,48 @@ /* Execute a program and parse its output */ -void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) +void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.exec"); auto elems = args[0]->listElems(); auto count = args[0]->listSize(); - if (count == 0) { - throw EvalError({ - .msg = hintfmt("at least one argument to 'exec' required"), - .errPos = pos - }); - } - PathSet context; - auto program = state.coerceToString(pos, *elems[0], context, false, false).toOwned(); + if (count == 0) + state.error("at least one argument to 'exec' required").atPos(pos).debugThrow(); + NixStringContext context; + auto program = state.coerceToString(pos, *elems[0], context, + "while evaluating the first element of the argument passed to builtins.exec", + false, false).toOwned(); Strings commandArgs; for (unsigned int i = 1; i < args[0]->listSize(); ++i) { - commandArgs.push_back(state.coerceToString(pos, *elems[i], context, false, false).toOwned()); + commandArgs.push_back( + state.coerceToString(pos, *elems[i], context, + "while evaluating an element of the argument passed to builtins.exec", + false, false).toOwned()); } try { auto _ = state.realiseContext(context); // FIXME: Handle CA derivations } catch (InvalidPathError & e) { - throw EvalError({ - .msg = hintfmt("cannot execute '%1%', since path '%2%' is not valid", - program, e.path), - .errPos = pos - }); + state.error("cannot execute '%1%', since path '%2%' is not valid", program, e.path).atPos(pos).debugThrow(); } auto output = runProgram(program, true, commandArgs); Expr * parsed; try { - parsed = state.parseExprFromString(std::move(output), pos.file); + parsed = state.parseExprFromString(std::move(output), state.rootPath(CanonPath::root)); } catch (Error & e) { - e.addTrace(pos, "While parsing the output from '%1%'", program); + e.addTrace(state.positions[pos], "while parsing the output from '%1%'", program); throw; } try { state.eval(parsed, v); } catch (Error & e) { - e.addTrace(pos, "While evaluating the output from '%1%'", program); + e.addTrace(state.positions[pos], "while evaluating the output from '%1%'", program); throw; } } - /* Return a string representing the type of the expression. */ -static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_typeOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); std::string t; @@ -402,7 +409,7 @@ case nFloat: t = "float"; break; case nThunk: abort(); } - v.mkString(state.symbols.create(t)); + v.mkString(t); } static RegisterPrimOp primop_typeOf({ @@ -417,7 +424,7 @@ }); /* Determine whether the argument is the null value. */ -static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_isNull(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); v.mkBool(args[0]->type() == nNull); @@ -437,7 +444,7 @@ }); /* Determine whether the argument is a function. */ -static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_isFunction(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); v.mkBool(args[0]->type() == nFunction); @@ -453,7 +460,7 @@ }); /* Determine whether the argument is an integer. */ -static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_isInt(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); v.mkBool(args[0]->type() == nInt); @@ -469,7 +476,7 @@ }); /* Determine whether the argument is a float. */ -static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_isFloat(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); v.mkBool(args[0]->type() == nFloat); @@ -485,7 +492,7 @@ }); /* Determine whether the argument is a string. */ -static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_isString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); v.mkBool(args[0]->type() == nString); @@ -501,7 +508,7 @@ }); /* Determine whether the argument is a Boolean. */ -static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_isBool(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); v.mkBool(args[0]->type() == nBool); @@ -517,7 +524,7 @@ }); /* Determine whether the argument is a path. */ -static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_isPath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); v.mkBool(args[0]->type() == nPath); @@ -532,49 +539,80 @@ .fun = prim_isPath, }); +template + static inline void withExceptionContext(Trace trace, Callable&& func) +{ + try + { + func(); + } + catch(Error & e) + { + e.pushTrace(trace); + throw; + } +} + struct CompareValues { EvalState & state; + const PosIdx pos; + const std::string_view errorCtx; - CompareValues(EvalState & state) : state(state) { }; + CompareValues(EvalState & state, const PosIdx pos, const std::string_view && errorCtx) : state(state), pos(pos), errorCtx(errorCtx) { }; bool operator () (Value * v1, Value * v2) const { - if (v1->type() == nFloat && v2->type() == nInt) - return v1->fpoint < v2->integer; - if (v1->type() == nInt && v2->type() == nFloat) - return v1->integer < v2->fpoint; - if (v1->type() != v2->type()) - throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); - switch (v1->type()) { - case nInt: - return v1->integer < v2->integer; - case nFloat: - return v1->fpoint < v2->fpoint; - case nString: - return strcmp(v1->string.s, v2->string.s) < 0; - case nPath: - return strcmp(v1->path, v2->path) < 0; - case nList: - // Lexicographic comparison - for (size_t i = 0;; i++) { - if (i == v2->listSize()) { - return false; - } else if (i == v1->listSize()) { - return true; - } else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i])) { - return (*this)(v1->listElems()[i], v2->listElems()[i]); + return (*this)(v1, v2, errorCtx); + } + + bool operator () (Value * v1, Value * v2, std::string_view errorCtx) const + { + try { + if (v1->type() == nFloat && v2->type() == nInt) + return v1->fpoint < v2->integer; + if (v1->type() == nInt && v2->type() == nFloat) + return v1->integer < v2->fpoint; + if (v1->type() != v2->type()) + state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" + switch (v1->type()) { + case nInt: + return v1->integer < v2->integer; + case nFloat: + return v1->fpoint < v2->fpoint; + case nString: + return strcmp(v1->string.s, v2->string.s) < 0; + case nPath: + return strcmp(v1->_path, v2->_path) < 0; + case nList: + // Lexicographic comparison + for (size_t i = 0;; i++) { + if (i == v2->listSize()) { + return false; + } else if (i == v1->listSize()) { + return true; + } else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i], pos, errorCtx)) { + return (*this)(v1->listElems()[i], v2->listElems()[i], "while comparing two list elements"); + } } - } - default: - throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); + default: + state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow(); + #pragma GCC diagnostic pop + } + } catch (Error & e) { + if (!errorCtx.empty()) + e.addTrace(nullptr, errorCtx); + throw; } } }; #if HAVE_BOEHMGC -typedef std::list > ValueList; +typedef std::list> ValueList; #else typedef std::list ValueList; #endif @@ -582,105 +620,67 @@ static Bindings::iterator getAttr( EvalState & state, - std::string_view funcName, Symbol attrSym, Bindings * attrSet, - const Pos & pos) + std::string_view errorCtx) { Bindings::iterator value = attrSet->find(attrSym); if (value == attrSet->end()) { - hintformat errorMsg = hintfmt( - "attribute '%s' missing for call to '%s'", - attrSym, - funcName - ); - - Pos aPos = *attrSet->pos; - if (aPos == noPos) { - throw TypeError({ - .msg = errorMsg, - .errPos = pos, - }); - } else { - auto e = TypeError({ - .msg = errorMsg, - .errPos = aPos, - }); - - // Adding another trace for the function name to make it clear - // which call received wrong arguments. - e.addTrace(pos, hintfmt("while invoking '%s'", funcName)); - throw e; - } + state.error("attribute '%s' missing", state.symbols[attrSym]).withTrace(noPos, errorCtx).debugThrow(); } - return value; } -static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], noPos, "while evaluating the first argument passed to builtins.genericClosure"); /* Get the start set. */ - Bindings::iterator startSet = getAttr( - state, - "genericClosure", - state.sStartSet, - args[0]->attrs, - pos - ); + Bindings::iterator startSet = getAttr(state, state.sStartSet, args[0]->attrs, "in the attrset passed as argument to builtins.genericClosure"); - state.forceList(*startSet->value, pos); + state.forceList(*startSet->value, noPos, "while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure"); ValueList workSet; for (auto elem : startSet->value->listItems()) workSet.push_back(elem); - /* Get the operator. */ - Bindings::iterator op = getAttr( - state, - "genericClosure", - state.sOperator, - args[0]->attrs, - pos - ); + if (startSet->value->listSize() == 0) { + v = *startSet->value; + return; + } - state.forceValue(*op->value, pos); + /* Get the operator. */ + Bindings::iterator op = getAttr(state, state.sOperator, args[0]->attrs, "in the attrset passed as argument to builtins.genericClosure"); + state.forceFunction(*op->value, noPos, "while evaluating the 'operator' attribute passed as argument to builtins.genericClosure"); - /* Construct the closure by applying the operator to element of + /* Construct the closure by applying the operator to elements of `workSet', adding the result to `workSet', continuing until no new elements are found. */ ValueList res; // `doneKeys' doesn't need to be a GC root, because its values are // reachable from res. - auto cmp = CompareValues(state); + auto cmp = CompareValues(state, noPos, "while comparing the `key` attributes of two genericClosure elements"); std::set doneKeys(cmp); while (!workSet.empty()) { Value * e = *(workSet.begin()); workSet.pop_front(); - state.forceAttrs(*e, pos); + state.forceAttrs(*e, noPos, "while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure"); - Bindings::iterator key = - e->attrs->find(state.sKey); - if (key == e->attrs->end()) - throw EvalError({ - .msg = hintfmt("attribute 'key' required"), - .errPos = pos - }); - state.forceValue(*key->value, pos); + Bindings::iterator key = getAttr(state, state.sKey, e->attrs, "in one of the attrsets generated by (or initially passed to) builtins.genericClosure"); + state.forceValue(*key->value, noPos); if (!doneKeys.insert(key->value).second) continue; res.push_back(e); /* Call the `operator' function with `e' as argument. */ - Value call; - call.mkApp(op->value, e); - state.forceList(call, pos); + Value newElements; + state.callFunction(*op->value, 1, &e, newElements, noPos); + state.forceList(newElements, noPos, "while evaluating the return value of the `operator` passed to builtins.genericClosure"); /* Add the values returned by the operator to the work set. */ - for (auto elem : call.listItems()) { - state.forceValue(*elem, pos); + for (auto elem : newElements.listItems()) { + state.forceValue(*elem, noPos); // "while evaluating one one of the elements returned by the `operator` passed to builtins.genericClosure"); workSet.push_back(elem); } } @@ -698,12 +698,14 @@ .arity = 1, .doc = R"( Take an *attrset* with values named `startSet` and `operator` in order to - return a *list of attrsets* by starting with the `startSet`, recursively - applying the `operator` function to each element. The *attrsets* in the - `startSet` and produced by the `operator` must each contain value named - `key` which are comparable to each other. The result is produced by - repeatedly calling the operator for each element encountered with a - unique key, terminating when no new elements are produced. For example, + return a *list of attrsets* by starting with the `startSet` and recursively + applying the `operator` function to each `item`. The *attrsets* in the + `startSet` and the *attrsets* produced by `operator` must contain a value + named `key` which is comparable. The result is produced by calling `operator` + for each `item` with a value for `key` that has not been called yet including + newly produced `item`s. The function terminates when no new `item`s are + produced. The resulting *list of attrsets* contains only *attrsets* with a + unique key. For example, ``` builtins.genericClosure { @@ -723,17 +725,53 @@ .fun = prim_genericClosure, }); + +static RegisterPrimOp primop_break({ + .name = "break", + .args = {"v"}, + .doc = R"( + In debug mode (enabled using `--debugger`), pause Nix expression evaluation and enter the REPL. + Otherwise, return the argument `v`. + )", + .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) + { + if (state.debugRepl && !state.debugTraces.empty()) { + auto error = Error(ErrorInfo { + .level = lvlInfo, + .msg = hintfmt("breakpoint reached"), + .errPos = state.positions[pos], + }); + + auto & dt = state.debugTraces.front(); + state.runDebugRepl(&error, dt.env, dt.expr); + + if (state.debugQuit) { + // If the user elects to quit the repl, throw an exception. + throw Error(ErrorInfo{ + .level = lvlInfo, + .msg = hintfmt("quit the debugger"), + .errPos = nullptr, + }); + } + } + + // Return the value we were passed. + v = *args[0]; + } +}); + static RegisterPrimOp primop_abort({ .name = "abort", .args = {"s"}, .doc = R"( Abort Nix expression evaluation and print the error message *s*. )", - .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) + .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto s = state.coerceToString(pos, *args[0], context).toOwned(); - throw Abort("evaluation aborted with the following error message: '%1%'", s); + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, + "while evaluating the error message passed to builtins.abort").toOwned(); + state.debugThrowLastTrace(Abort("evaluation aborted with the following error message: '%1%'", s)); } }); @@ -747,22 +785,26 @@ derivations, a derivation that throws an error is silently skipped (which is not the case for `abort`). )", - .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) + .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto s = state.coerceToString(pos, *args[0], context).toOwned(); - throw ThrownError(s); + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, + "while evaluating the error message passed to builtin.throw").toOwned(); + state.debugThrowLastTrace(ThrownError(s)); } }); -static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { try { state.forceValue(*args[1], pos); v = *args[1]; } catch (Error & e) { - PathSet context; - e.addTrace(std::nullopt, state.coerceToString(pos, *args[0], context).toOwned()); + NixStringContext context; + auto message = state.coerceToString(pos, *args[0], context, + "while evaluating the error message passed to builtins.addErrorContext", + false, false).toOwned(); + e.addTrace(nullptr, message, true); throw; } } @@ -773,9 +815,10 @@ .fun = prim_addErrorContext, }); -static void prim_ceil(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_ceil(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos), + "while evaluating the first argument passed to builtins.ceil"); v.mkInt(ceil(value)); } @@ -792,9 +835,9 @@ .fun = prim_ceil, }); -static void prim_floor(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_floor(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos), "while evaluating the first argument passed to builtins.floor"); v.mkInt(floor(value)); } @@ -813,9 +856,21 @@ /* Try evaluating the argument. Success => {success=true; value=something;}, * else => {success=false; value=false;} */ -static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto attrs = state.buildBindings(2); + + /* increment state.trylevel, and decrement it when this function returns. */ + MaintainCount trylevel(state.trylevel); + + void (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; + if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry) + { + /* to prevent starting the repl from exceptions withing a tryEval, null it. */ + savedDebugRepl = state.debugRepl; + state.debugRepl = nullptr; + } + try { state.forceValue(*args[0], pos); attrs.insert(state.sValue, args[0]); @@ -824,6 +879,11 @@ attrs.alloc(state.sValue).mkBool(false); attrs.alloc("success").mkBool(false); } + + // restore the debugRepl pointer if we saved it earlier. + if (savedDebugRepl) + state.debugRepl = savedDebugRepl; + v.mkAttrs(attrs); } @@ -849,9 +909,9 @@ }); /* Return an environment variable. Use with care. */ -static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_getEnv(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - std::string name(state.forceStringNoCtx(*args[0], pos)); + std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv")); v.mkString(evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); } @@ -873,7 +933,7 @@ }); /* Evaluate the first argument, then return the second argument. */ -static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_seq(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); @@ -892,7 +952,7 @@ /* Evaluate the first argument deeply (i.e. recursing into lists and attrsets), then return the second argument. */ -static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_deepSeq(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValueDeep(*args[0]); state.forceValue(*args[1], pos); @@ -912,13 +972,13 @@ /* Evaluate the first expression and print it on standard error. Then return the second expression. Useful for debugging. */ -static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); if (args[0]->type() == nString) printError("trace: %1%", args[0]->string.s); else - printError("trace: %1%", *args[0]); + printError("trace: %1%", printValue(state, *args[0])); state.forceValue(*args[1], pos); v = *args[1]; } @@ -935,10 +995,20 @@ }); +/* Takes two arguments and evaluates to the second one. Used as the + * builtins.traceVerbose implementation when --trace-verbose is not enabled + */ +static void prim_second(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + state.forceValue(*args[1], pos); + v = *args[1]; +} + /************************************************************* * Derivations *************************************************************/ +static void derivationStrictInternal(EvalState & state, const std::string & name, Bindings * attrs, Value & v); /* Construct (as a unobservable side effect) a Nix derivation expression that performs the derivation described by the argument @@ -947,123 +1017,159 @@ derivation; `drvPath' containing the path of the Nix expression; and `type' set to `derivation' to indicate that this is a derivation. */ -static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.derivationStrict"); + + Bindings * attrs = args[0]->attrs; /* Figure out the name first (for stack backtraces). */ - Bindings::iterator attr = getAttr( - state, - "derivationStrict", - state.sName, - args[0]->attrs, - pos - ); + Bindings::iterator nameAttr = getAttr(state, state.sName, attrs, "in the attrset passed as argument to builtins.derivationStrict"); std::string drvName; - Pos & posDrvName(*attr->pos); try { - drvName = state.forceStringNoCtx(*attr->value, pos); + drvName = state.forceStringNoCtx(*nameAttr->value, pos, "while evaluating the `name` attribute passed to builtins.derivationStrict"); + } catch (Error & e) { + e.addTrace(state.positions[nameAttr->pos], "while evaluating the derivation attribute 'name'"); + throw; + } + + try { + derivationStrictInternal(state, drvName, attrs, v); } catch (Error & e) { - e.addTrace(posDrvName, "while evaluating the derivation attribute 'name'"); + Pos pos = state.positions[nameAttr->pos]; + /* + * Here we make two abuses of the error system + * + * 1. We print the location as a string to avoid a code snippet being + * printed. While the location of the name attribute is a good hint, the + * exact code there is irrelevant. + * + * 2. We mark this trace as a frame trace, meaning that we stop printing + * less important traces from now on. In particular, this prevents the + * display of the automatic "while calling builtins.derivationStrict" + * trace, which is of little use for the public we target here. + * + * Please keep in mind that error reporting is done on a best-effort + * basis in nix. There is no accurate location for a derivation, as it + * often results from the composition of several functions + * (derivationStrict, derivation, mkDerivation, mkPythonModule, etc.) + */ + e.addTrace(nullptr, hintfmt( + "while evaluating derivation '%s'\n" + " whose name attribute is located at %s", + drvName, pos), true); throw; } +} +static void derivationStrictInternal(EvalState & state, const std::string & +drvName, Bindings * attrs, Value & v) +{ /* Check whether attributes should be passed as a JSON file. */ - std::ostringstream jsonBuf; - std::unique_ptr jsonObject; - attr = args[0]->attrs->find(state.sStructuredAttrs); - if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos)) - jsonObject = std::make_unique(jsonBuf); + using nlohmann::json; + std::optional jsonObject; + auto attr = attrs->find(state.sStructuredAttrs); + if (attr != attrs->end() && + state.forceBool(*attr->value, noPos, + "while evaluating the `__structuredAttrs` " + "attribute passed to builtins.derivationStrict")) + jsonObject = json::object(); /* Check whether null attributes should be ignored. */ bool ignoreNulls = false; - attr = args[0]->attrs->find(state.sIgnoreNulls); - if (attr != args[0]->attrs->end()) - ignoreNulls = state.forceBool(*attr->value, pos); + attr = attrs->find(state.sIgnoreNulls); + if (attr != attrs->end()) + ignoreNulls = state.forceBool(*attr->value, noPos, "while evaluating the `__ignoreNulls` attribute " "passed to builtins.derivationStrict"); /* Build the derivation expression by processing the attributes. */ Derivation drv; drv.name = drvName; - PathSet context; + NixStringContext context; bool contentAddressed = false; bool isImpure = false; std::optional outputHash; std::string outputHashAlgo; - std::optional ingestionMethod; + std::optional ingestionMethod; StringSet outputs; outputs.insert("out"); - for (auto & i : args[0]->attrs->lexicographicOrder()) { + for (auto & i : attrs->lexicographicOrder(state.symbols)) { if (i->name == state.sIgnoreNulls) continue; - const std::string & key = i->name; + const std::string & key = state.symbols[i->name]; vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string_view s) { if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; - else - throw EvalError({ + else if (s == "text") { + experimentalFeatureSettings.require(Xp::DynamicDerivations); + ingestionMethod = TextIngestionMethod {}; + } else + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), - .errPos = posDrvName - }); + .errPos = state.positions[noPos] + })); }; auto handleOutputs = [&](const Strings & ss) { outputs.clear(); for (auto & j : ss) { if (outputs.find(j) != outputs.end()) - throw EvalError({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("duplicate derivation output '%1%'", j), - .errPos = posDrvName - }); + .errPos = state.positions[noPos] + })); /* !!! Check whether j is a valid attribute name. */ /* Derivations cannot be named ‘drv’, because then we'd have an attribute ‘drvPath’ in the resulting set. */ if (j == "drv") - throw EvalError({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("invalid derivation output name 'drv'" ), - .errPos = posDrvName - }); + .errPos = state.positions[noPos] + })); outputs.insert(j); } if (outputs.empty()) - throw EvalError({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("derivation cannot have an empty set of outputs"), - .errPos = posDrvName - }); + .errPos = state.positions[noPos] + })); }; try { + // This try-catch block adds context for most errors. + // Use this empty error context to signify that we defer to it. + const std::string_view context_below(""); if (ignoreNulls) { - state.forceValue(*i->value, pos); + state.forceValue(*i->value, noPos); if (i->value->type() == nNull) continue; } - if (i->name == state.sContentAddressed) { - contentAddressed = state.forceBool(*i->value, pos); - if (contentAddressed) - settings.requireExperimentalFeature(Xp::CaDerivations); + if (i->name == state.sContentAddressed && state.forceBool(*i->value, noPos, context_below)) { + contentAddressed = true; + experimentalFeatureSettings.require(Xp::CaDerivations); } - else if (i->name == state.sImpure) { - isImpure = state.forceBool(*i->value, pos); - if (isImpure) - settings.requireExperimentalFeature(Xp::ImpureDerivations); + else if (i->name == state.sImpure && state.forceBool(*i->value, noPos, context_below)) { + isImpure = true; + experimentalFeatureSettings.require(Xp::ImpureDerivations); } /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ else if (i->name == state.sArgs) { - state.forceList(*i->value, pos); + state.forceList(*i->value, noPos, context_below); for (auto elem : i->value->listItems()) { - auto s = state.coerceToString(posDrvName, *elem, context, true).toOwned(); + auto s = state.coerceToString(noPos, *elem, context, + "while evaluating an element of the argument list", + true).toOwned(); drv.args.push_back(s); } } @@ -1076,30 +1182,29 @@ if (i->name == state.sStructuredAttrs) continue; - auto placeholder(jsonObject->placeholder(key)); - printValueAsJSON(state, true, *i->value, pos, placeholder, context); + (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, noPos, context); if (i->name == state.sBuilder) - drv.builder = state.forceString(*i->value, context, posDrvName); + drv.builder = state.forceString(*i->value, context, noPos, context_below); else if (i->name == state.sSystem) - drv.platform = state.forceStringNoCtx(*i->value, posDrvName); + drv.platform = state.forceStringNoCtx(*i->value, noPos, context_below); else if (i->name == state.sOutputHash) - outputHash = state.forceStringNoCtx(*i->value, posDrvName); + outputHash = state.forceStringNoCtx(*i->value, noPos, context_below); else if (i->name == state.sOutputHashAlgo) - outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName); + outputHashAlgo = state.forceStringNoCtx(*i->value, noPos, context_below); else if (i->name == state.sOutputHashMode) - handleHashMode(state.forceStringNoCtx(*i->value, posDrvName)); + handleHashMode(state.forceStringNoCtx(*i->value, noPos, context_below)); else if (i->name == state.sOutputs) { /* Require ‘outputs’ to be a list of strings. */ - state.forceList(*i->value, posDrvName); + state.forceList(*i->value, noPos, context_below); Strings ss; for (auto elem : i->value->listItems()) - ss.emplace_back(state.forceStringNoCtx(*elem, posDrvName)); + ss.emplace_back(state.forceStringNoCtx(*elem, noPos, context_below)); handleOutputs(ss); } } else { - auto s = state.coerceToString(*i->pos, *i->value, context, true).toOwned(); + auto s = state.coerceToString(noPos, *i->value, context, context_below, true).toOwned(); drv.env.emplace(key, s); if (i->name == state.sBuilder) drv.builder = std::move(s); else if (i->name == state.sSystem) drv.platform = std::move(s); @@ -1113,71 +1218,71 @@ } } catch (Error & e) { - e.addTrace(posDrvName, - "while evaluating the attribute '%1%' of the derivation '%2%'", - key, drvName); + e.addTrace(state.positions[i->pos], + hintfmt("while evaluating attribute '%1%' of derivation '%2%'", key, drvName), + true); throw; } } if (jsonObject) { + drv.env.emplace("__json", jsonObject->dump()); jsonObject.reset(); - drv.env.emplace("__json", jsonBuf.str()); } /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ - for (auto & path : context) { - - /* Paths marked with `=' denote that the path of a derivation - is explicitly passed to the builder. Since that allows the - builder to gain access to every path in the dependency - graph of the derivation (including all outputs), all paths - in the graph must be added to this derivation's list of - inputs to ensure that they are available when the builder - runs. */ - if (path.at(0) == '=') { - /* !!! This doesn't work if readOnlyMode is set. */ - StorePathSet refs; - state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs); - for (auto & j : refs) { - drv.inputSrcs.insert(j); - if (j.isDerivation()) - drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); - } - } - - /* Handle derivation outputs of the form ‘!!’. */ - else if (path.at(0) == '!') { - auto ctx = decodeContext(*state.store, path); - drv.inputDrvs[ctx.first].insert(ctx.second); - } - - /* Otherwise it's a source file. */ - else - drv.inputSrcs.insert(state.store->parseStorePath(path)); + for (auto & c : context) { + std::visit(overloaded { + /* Since this allows the builder to gain access to every + path in the dependency graph of the derivation (including + all outputs), all paths in the graph must be added to + this derivation's list of inputs to ensure that they are + available when the builder runs. */ + [&](const NixStringContextElem::DrvDeep & d) { + /* !!! This doesn't work if readOnlyMode is set. */ + StorePathSet refs; + state.store->computeFSClosure(d.drvPath, refs); + for (auto & j : refs) { + drv.inputSrcs.insert(j); + if (j.isDerivation()) + drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); + } + }, + [&](const NixStringContextElem::Built & b) { + drv.inputDrvs[b.drvPath].insert(b.output); + }, + [&](const NixStringContextElem::Opaque & o) { + drv.inputSrcs.insert(o.path); + }, + }, c.raw()); } /* Do we have all required attributes? */ if (drv.builder == "") - throw EvalError({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("required attribute 'builder' missing"), - .errPos = posDrvName - }); + .errPos = state.positions[noPos] + })); if (drv.platform == "") - throw EvalError({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("required attribute 'system' missing"), - .errPos = posDrvName - }); + .errPos = state.positions[noPos] + })); /* Check whether the derivation name is valid. */ - if (isDerivation(drvName)) - throw EvalError({ - .msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), - .errPos = posDrvName - }); + if (isDerivation(drvName) && + !(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } && + outputs.size() == 1 && + *(outputs.begin()) == "out")) + { + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension), + .errPos = state.positions[noPos] + })); + } if (outputHash) { /* Handle fixed-output derivations. @@ -1185,30 +1290,30 @@ Ignore `__contentAddressed` because fixed output derivations are already content addressed. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") - throw Error({ + state.debugThrowLastTrace(Error({ .msg = hintfmt("multiple outputs are not supported in fixed-output derivations"), - .errPos = posDrvName - }); + .errPos = state.positions[noPos] + })); auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); - auto outPath = state.store->makeFixedOutputPath(method, h, drvName); - drv.env["out"] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", - DerivationOutput::CAFixed { - .hash = FixedOutputHash { - .method = method, - .hash = std::move(h), - }, - }); + + DerivationOutput::CAFixed dof { + .ca = ContentAddress::fromParts( + std::move(method), + std::move(h)), + }; + + drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); + drv.outputs.insert_or_assign("out", std::move(dof)); } else if (contentAddressed || isImpure) { if (contentAddressed && isImpure) throw EvalError({ .msg = hintfmt("derivation cannot be both content-addressed and impure"), - .errPos = posDrvName + .errPos = state.positions[noPos] }); auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256); @@ -1219,13 +1324,13 @@ if (isImpure) drv.outputs.insert_or_assign(i, DerivationOutput::Impure { - .method = method, + .method = method.raw, .hashType = ht, }); else drv.outputs.insert_or_assign(i, DerivationOutput::CAFloating { - .method = method, + .method = method.raw, .hashType = ht, }); } @@ -1248,8 +1353,13 @@ switch (hashModulo.kind) { case DrvHash::Kind::Regular: for (auto & i : outputs) { - auto h = hashModulo.hashes.at(i); - auto outPath = state.store->makeOutputPath(i, h, drvName); + auto h = get(hashModulo.hashes, i); + if (!h) + throw AssertionError({ + .msg = hintfmt("derivation produced no hash for output '%s'", i), + .errPos = state.positions[noPos], + }); + auto outPath = state.store->makeOutputPath(i, *h, drvName); drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign( i, @@ -1280,11 +1390,14 @@ drvHashes.lock()->insert_or_assign(drvPath, h); } - auto attrs = state.buildBindings(1 + drv.outputs.size()); - attrs.alloc(state.sDrvPath).mkString(drvPathS, {"=" + drvPathS}); + auto result = state.buildBindings(1 + drv.outputs.size()); + result.alloc(state.sDrvPath).mkString(drvPathS, { + NixStringContextElem::DrvDeep { .drvPath = drvPath }, + }); for (auto & i : drv.outputs) - mkOutputString(state, attrs, drvPath, drv, i); - v.mkAttrs(attrs); + mkOutputString(state, result, drvPath, i); + + v.mkAttrs(result); } static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { @@ -1300,9 +1413,9 @@ time, any occurrence of this string in an derivation attribute will be replaced with the concrete path in the Nix store of the output ‘out’. */ -static void prim_placeholder(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_placeholder(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkString(hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); + v.mkString(hashPlaceholder(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.placeholder"))); } static RegisterPrimOp primop_placeholder({ @@ -1323,11 +1436,11 @@ /* Convert the argument to a path. !!! obsolete? */ -static void prim_toPath(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_toPath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); - v.mkString(canonPath(path), context); + NixStringContext context; + auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath"); + v.mkString(path.path.abs(), context); } static RegisterPrimOp primop_toPath({ @@ -1348,30 +1461,31 @@ /nix/store/newhash-oldhash-oldname. In the past, `toPath' had special case behaviour for store paths, but that created weird corner cases. */ -static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { if (evalSettings.pureEval) - throw EvalError({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("'%s' is not allowed in pure evaluation mode", "builtins.storePath"), - .errPos = pos - }); + .errPos = state.positions[pos] + })); - PathSet context; - Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context)); + NixStringContext context; + auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path; /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ - if (!state.store->isStorePath(path)) path = canonPath(path, true); - if (!state.store->isInStore(path)) - throw EvalError({ + if (!state.store->isStorePath(path.abs())) + path = CanonPath(canonPath(path.abs(), true)); + if (!state.store->isInStore(path.abs())) + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("path '%1%' is not in the Nix store", path), - .errPos = pos - }); - auto path2 = state.store->toStorePath(path).first; + .errPos = state.positions[pos] + })); + auto path2 = state.store->toStorePath(path.abs()).first; if (!settings.readOnlyMode) state.store->ensurePath(path2); - context.insert(state.store->printStorePath(path2)); - v.mkString(path, context); + context.insert(NixStringContextElem::Opaque { .path = path2 }); + v.mkString(path.abs(), context); } static RegisterPrimOp primop_storePath({ @@ -1387,22 +1501,22 @@ causes the path to be *copied* again to the Nix store, resulting in a new path (e.g. `/nix/store/ld01dnzc…-source-source`). - This function is not available in pure evaluation mode. + Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). )", .fun = prim_storePath, }); -static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v) { /* We don’t check the path right now, because we don’t want to - throw if the path isn’t allowed, but just return false (and we - can’t just catch the exception here because we still want to - throw if something in the evaluation of `*args[0]` tries to - access an unauthorized path). */ + throw if the path isn’t allowed, but just return false (and we + can’t just catch the exception here because we still want to + throw if something in the evaluation of `*args[0]` tries to + access an unauthorized path). */ auto path = realisePath(state, pos, *args[0], { .checkForPureEval = false }); try { - v.mkBool(pathExists(state.checkSourcePath(path))); + v.mkBool(state.checkSourcePath(path).pathExists()); } catch (SysError & e) { /* Don't give away info from errors while canonicalising ‘path’ in restricted mode. */ @@ -1424,10 +1538,12 @@ /* Return the base name of the given string, i.e., everything following the last slash. */ -static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - v.mkString(baseNameOf(*state.coerceToString(pos, *args[0], context, false, false)), context); + NixStringContext context; + v.mkString(baseNameOf(*state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to builtins.baseNameOf", + false, false)), context); } static RegisterPrimOp primop_baseNameOf({ @@ -1444,12 +1560,20 @@ /* Return the directory of the given path, i.e., everything before the last slash. Return either a path or a string depending on the type of the argument. */ -static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_dirOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto path = state.coerceToString(pos, *args[0], context, false, false); - auto dir = dirOf(*path); - if (args[0]->type() == nPath) v.mkPath(dir); else v.mkString(dir, context); + state.forceValue(*args[0], pos); + if (args[0]->type() == nPath) { + auto path = args[0]->path(); + v.mkPath(path.path.isRoot() ? path : path.parent()); + } else { + NixStringContext context; + auto path = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to 'builtins.dirOf'", + false, false); + auto dir = dirOf(*path); + v.mkString(dir, context); + } } static RegisterPrimOp primop_dirOf({ @@ -1464,20 +1588,29 @@ }); /* Return the contents of a file as a string. */ -static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto path = realisePath(state, pos, *args[0]); - auto s = readFile(path); + auto s = path.readFile(); if (s.find((char) 0) != std::string::npos) - throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path); + state.debugThrowLastTrace(Error("the contents of the file '%1%' cannot be represented as a Nix string", path)); StorePathSet refs; - if (state.store->isInStore(path)) { + if (state.store->isInStore(path.path.abs())) { try { - refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; + refs = state.store->queryPathInfo(state.store->toStorePath(path.path.abs()).first)->references; } catch (Error &) { // FIXME: should be InvalidPathError } + // Re-scan references to filter down to just the ones that actually occur in the file. + auto refsSink = PathRefScanSink::fromPaths(refs); + refsSink << s; + refs = refsSink.getResultPaths(); + } + NixStringContext context; + for (auto && p : std::move(refs)) { + context.insert(NixStringContextElem::Opaque { + .path = std::move((StorePath &&)p), + }); } - auto context = state.store->printStorePathSet(refs); v.mkString(s, context); } @@ -1492,46 +1625,41 @@ /* Find a file in the Nix search path. Used to implement paths, which are desugared to 'findFile __nixPath "x"'. */ -static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.findFile"); SearchPath searchPath; for (auto v2 : args[0]->listItems()) { - state.forceAttrs(*v2, pos); + state.forceAttrs(*v2, pos, "while evaluating an element of the list passed to builtins.findFile"); std::string prefix; Bindings::iterator i = v2->attrs->find(state.sPrefix); if (i != v2->attrs->end()) - prefix = state.forceStringNoCtx(*i->value, pos); + prefix = state.forceStringNoCtx(*i->value, pos, "while evaluating the `prefix` attribute of an element of the list passed to builtins.findFile"); - i = getAttr( - state, - "findFile", - state.sPath, - v2->attrs, - pos - ); + i = getAttr(state, state.sPath, v2->attrs, "in an element of the __nixPath"); - PathSet context; - auto path = state.coerceToString(pos, *i->value, context, false, false).toOwned(); + NixStringContext context; + auto path = state.coerceToString(pos, *i->value, context, + "while evaluating the `path` attribute of an element of the list passed to builtins.findFile", + false, false).toOwned(); try { auto rewrites = state.realiseContext(context); path = rewriteStrings(path, rewrites); } catch (InvalidPathError & e) { - throw EvalError({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), - .errPos = pos - }); + .errPos = state.positions[pos] + })); } - searchPath.emplace_back(prefix, path); } - auto path = state.forceStringNoCtx(*args[1], pos); + auto path = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile"); v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos))); } @@ -1543,19 +1671,19 @@ }); /* Return the cryptographic hash of a file in base-16. */ -static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto type = state.forceStringNoCtx(*args[0], pos); + auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile"); std::optional ht = parseHashType(type); if (!ht) - throw Error({ + state.debugThrowLastTrace(Error({ .msg = hintfmt("unknown hash type '%1%'", type), - .errPos = pos - }); + .errPos = state.positions[pos] + })); auto path = realisePath(state, pos, *args[1]); - v.mkString(hashFile(*ht, path).to_string(Base16, false)); + v.mkString(hashString(*ht, path.readFile()).to_string(Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -1569,23 +1697,65 @@ .fun = prim_hashFile, }); -/* Read a directory (without . or ..) */ -static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) +static std::string_view fileTypeToString(InputAccessor::Type type) +{ + return + type == InputAccessor::Type::tRegular ? "regular" : + type == InputAccessor::Type::tDirectory ? "directory" : + type == InputAccessor::Type::tSymlink ? "symlink" : + "unknown"; +} + +static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto path = realisePath(state, pos, *args[0]); + /* Retrieve the directory entry type and stringize it. */ + v.mkString(fileTypeToString(path.lstat().type)); +} - DirEntries entries = readDirectory(path); +static RegisterPrimOp primop_readFileType({ + .name = "__readFileType", + .args = {"p"}, + .doc = R"( + Determine the directory entry type of a filesystem node, being + one of "directory", "regular", "symlink", or "unknown". + )", + .fun = prim_readFileType, +}); +/* Read a directory (without . or ..) */ +static void prim_readDir(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + auto path = realisePath(state, pos, *args[0]); + + // Retrieve directory entries for all nodes in a directory. + // This is similar to `getFileType` but is optimized to reduce system calls + // on many systems. + auto entries = path.readDirectory(); auto attrs = state.buildBindings(entries.size()); - for (auto & ent : entries) { - if (ent.type == DT_UNKNOWN) - ent.type = getFileType(path + "/" + ent.name); - attrs.alloc(ent.name).mkString( - ent.type == DT_REG ? "regular" : - ent.type == DT_DIR ? "directory" : - ent.type == DT_LNK ? "symlink" : - "unknown"); + // If we hit unknown directory entry types we may need to fallback to + // using `getFileType` on some systems. + // In order to reduce system calls we make each lookup lazy by using + // `builtins.readFileType` application. + Value * readFileType = nullptr; + + for (auto & [name, type] : entries) { + auto & attr = attrs.alloc(name); + if (!type) { + // Some filesystems or operating systems may not be able to return + // detailed node info quickly in this case we produce a thunk to + // query the file type lazily. + auto epath = state.allocValue(); + epath->mkPath(path + name); + if (!readFileType) + readFileType = &state.getBuiltin("readFileType"); + attr.mkApp(readFileType, epath); + } else { + // This branch of the conditional is much more likely. + // Here we just stringize the directory entry type. + attr.mkString(fileTypeToString(*type)); + } } v.mkAttrs(attrs); @@ -1619,10 +1789,10 @@ /* Convert the argument (which can be any Nix expression) to an XML representation returned in a string. Not all Nix expressions can be sensibly or completely represented (e.g., functions). */ -static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_toXML(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::ostringstream out; - PathSet context; + NixStringContext context; printValueAsXML(state, true, false, *args[0], out, context, pos); v.mkString(out.str(), context); } @@ -1727,10 +1897,10 @@ /* Convert the argument (which can be any Nix expression) to a JSON string. Not all Nix expressions can be sensibly or completely represented (e.g., functions). */ -static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_toJSON(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::ostringstream out; - PathSet context; + NixStringContext context; printValueAsJSON(state, true, *args[0], pos, out, context); v.mkString(out.str(), context); } @@ -1750,13 +1920,13 @@ }); /* Parse a JSON string to a value. */ -static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_fromJSON(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto s = state.forceStringNoCtx(*args[0], pos); + auto s = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.fromJSON"); try { parseJSON(state, s, v); } catch (JSONParseError &e) { - e.addTrace(pos, "while decoding a JSON string"); + e.addTrace(state.positions[pos], "while decoding a JSON string"); throw; } } @@ -1778,35 +1948,37 @@ /* Store a string in the Nix store as a source file that can be used as an input by derivations. */ -static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - std::string name(state.forceStringNoCtx(*args[0], pos)); - std::string contents(state.forceString(*args[1], context, pos)); + NixStringContext context; + std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.toFile")); + std::string contents(state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile")); StorePathSet refs; - for (auto path : context) { - if (path.at(0) != '/') - throw EvalError( { + for (auto c : context) { + if (auto p = std::get_if(&c)) + refs.insert(p->path); + else + state.debugThrowLastTrace(EvalError({ .msg = hintfmt( "in 'toFile': the file named '%1%' must not contain a reference " "to a derivation but contains (%2%)", - name, path), - .errPos = pos - }); - refs.insert(state.store->parseStorePath(path)); + name, c.to_string()), + .errPos = state.positions[pos] + })); } - auto storePath = state.store->printStorePath(settings.readOnlyMode + auto storePath = settings.readOnlyMode ? state.store->computeStorePathForText(name, contents, refs) - : state.store->addTextToStore(name, contents, refs, state.repair)); + : state.store->addTextToStore(name, contents, refs, state.repair); /* Note: we don't need to add `context' to the context of the result, since `storePath' itself has references to the paths used in args[1]. */ - v.mkString(storePath, {storePath}); + /* Add the output of this to the allowed paths. */ + state.allowAndSetStorePathString(storePath, v); } static RegisterPrimOp primop_toFile({ @@ -1817,8 +1989,7 @@ path. The file has suffix *name*. This file can be used as an input to derivations. One application is to write builders “inline”. For instance, the following Nix expression combines the - [Nix expression for GNU Hello](expression-syntax.md) and its - [build script](build-script.md) into one file: + Nix expression for GNU Hello and its build script into one file: ```nix { stdenv, fetchurl, perl }: @@ -1861,8 +2032,8 @@ "; ``` - Note that `${configFile}` is an - [antiquotation](language-values.md), so the result of the + Note that `${configFile}` is a + [string interpolation](@docroot@/language/values.md#type-string), so the result of the expression `configFile` (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be spliced into the resulting string. @@ -1889,14 +2060,14 @@ static void addPath( EvalState & state, - const Pos & pos, - const std::string & name, + const PosIdx pos, + std::string_view name, Path path, Value * filterFun, FileIngestionMethod method, const std::optional expectedHash, Value & v, - const PathSet & context) + const NixStringContext & context) { try { // FIXME: handle CA derivation outputs (where path needs to @@ -1918,7 +2089,7 @@ path = evalSettings.pureEval && expectedHash ? path - : state.checkSourcePath(path); + : state.checkSourcePath(CanonPath(path)).path.abs(); PathFilter filter = filterFun ? ([&](const Path & path) { auto st = lstat(path); @@ -1939,44 +2110,42 @@ Value res; state.callFunction(*filterFun, 2, args, res, pos); - return state.forceBool(res, pos); + return state.forceBool(res, pos, "while evaluating the return value of the path filter function"); }) : defaultPathFilter; std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + .hash = { + .method = method, + .hash = *expectedHash, + }, + .references = {}, + }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { StorePath dstPath = settings.readOnlyMode ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs); if (expectedHash && expectedStorePath != dstPath) - throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); + state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); } else state.allowAndSetStorePathString(*expectedStorePath, v); } catch (Error & e) { - e.addTrace(pos, "while adding path '%s'", path); + e.addTrace(state.positions[pos], "while adding path '%s'", path); throw; } } -static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); - - state.forceValue(*args[0], pos); - if (args[0]->type() != nFunction) - throw TypeError({ - .msg = hintfmt( - "first argument in call to 'filterSource' is not a function but %1%", - showType(*args[0])), - .errPos = pos - }); - - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); + NixStringContext context; + auto path = state.coerceToPath(pos, *args[1], context, + "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); + addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ @@ -2034,44 +2203,44 @@ .fun = prim_filterSource, }); -static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); - Path path; + std::optional path; std::string name; Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; std::optional expectedHash; - PathSet context; + NixStringContext context; + + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to 'builtins.path'"); for (auto & attr : *args[0]->attrs) { - auto & n(attr.name); + auto n = state.symbols[attr.name]; if (n == "path") - path = state.coerceToPath(*attr.pos, *attr.value, context); + path.emplace(state.coerceToPath(attr.pos, *attr.value, context, "while evaluating the 'path' attribute passed to 'builtins.path'")); else if (attr.name == state.sName) - name = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "filter") { - state.forceValue(*attr.value, pos); - filterFun = attr.value; - } else if (n == "recursive") - method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) }; + name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `name` attribute passed to builtins.path"); + else if (n == "filter") + state.forceFunction(*(filterFun = attr.value), attr.pos, "while evaluating the `filter` parameter passed to builtins.path"); + else if (n == "recursive") + method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") }; else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), htSHA256); else - throw EvalError({ - .msg = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name), - .errPos = *attr.pos - }); - } - if (path.empty()) - throw EvalError({ - .msg = hintfmt("'path' required"), - .errPos = pos - }); + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("unsupported argument '%1%' to 'addPath'", state.symbols[attr.name]), + .errPos = state.positions[attr.pos] + })); + } + if (!path) + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("missing required 'path' attribute in the first argument to builtins.path"), + .errPos = state.positions[pos] + })); if (name.empty()) - name = baseNameOf(path); + name = path->baseName(); - addPath(state, pos, name, path, filterFun, method, expectedHash, v, context); + addPath(state, pos, name, path->path.abs(), filterFun, method, expectedHash, v, context); } static RegisterPrimOp primop_path({ @@ -2117,15 +2286,15 @@ /* Return the names of the attributes in a set as a sorted list of strings. */ -static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.attrNames"); state.mkList(v, args[0]->attrs->size()); size_t n = 0; for (auto & i : *args[0]->attrs) - (v.listElems()[n++] = state.allocValue())->mkString(i.name); + (v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]); std::sort(v.listElems(), v.listElems() + n, [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); @@ -2144,9 +2313,9 @@ /* Return the values of the attributes in a set as a list, in the same order as attrNames. */ -static void prim_attrValues(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_attrValues(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.attrValues"); state.mkList(v, args[0]->attrs->size()); @@ -2155,8 +2324,9 @@ v.listElems()[n++] = (Value *) &i; std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { - std::string_view s1 = ((Attr *) v1)->name, s2 = ((Attr *) v2)->name; + [&](Value * v1, Value * v2) { + std::string_view s1 = state.symbols[((Attr *) v1)->name], + s2 = state.symbols[((Attr *) v2)->name]; return s1 < s2; }); @@ -2175,19 +2345,18 @@ }); /* Dynamic version of the `.' operator. */ -void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) +void prim_getAttr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto attr = state.forceStringNoCtx(*args[0], pos); - state.forceAttrs(*args[1], pos); + auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getAttr"); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.getAttr"); Bindings::iterator i = getAttr( state, - "getAttr", state.symbols.create(attr), args[1]->attrs, - pos + "in the attribute set under consideration" ); // !!! add to stack trace? - if (state.countCalls && *i->pos != noPos) state.attrSelects[*i->pos]++; + if (state.countCalls && i->pos) state.attrSelects[i->pos]++; state.forceValue(*i->value, pos); v = *i->value; } @@ -2205,10 +2374,10 @@ }); /* Return position information of the specified attribute. */ -static void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_unsafeGetAttrPos(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto attr = state.forceStringNoCtx(*args[0], pos); - state.forceAttrs(*args[1], pos); + auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.unsafeGetAttrPos"); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.unsafeGetAttrPos"); Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); if (i == args[1]->attrs->end()) v.mkNull(); @@ -2223,10 +2392,10 @@ }); /* Dynamic version of the `?' operator. */ -static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_hasAttr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto attr = state.forceStringNoCtx(*args[0], pos); - state.forceAttrs(*args[1], pos); + auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hasAttr"); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.hasAttr"); v.mkBool(args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); } @@ -2242,7 +2411,7 @@ }); /* Determine whether the argument is a set. */ -static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_isAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); v.mkBool(args[0]->type() == nAttrs); @@ -2257,10 +2426,10 @@ .fun = prim_isAttrs, }); -static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); - state.forceList(*args[1], pos); + state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.removeAttrs"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.removeAttrs"); /* Get the attribute names to be removed. We keep them as Attrs instead of Symbols so std::set_difference @@ -2268,7 +2437,7 @@ boost::container::small_vector names; names.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { - state.forceStringNoCtx(*elem, pos); + state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs"); names.emplace_back(state.symbols.create(elem->string.s), nullptr); } std::sort(names.begin(), names.end()); @@ -2305,36 +2474,24 @@ "nameN"; value = valueN;}] is transformed to {name1 = value1; ... nameN = valueN;}. In case of duplicate occurrences of the same name, the first takes precedence. */ -static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_listToAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the argument passed to builtins.listToAttrs"); auto attrs = state.buildBindings(args[0]->listSize()); std::set seen; for (auto v2 : args[0]->listItems()) { - state.forceAttrs(*v2, pos); + state.forceAttrs(*v2, pos, "while evaluating an element of the list passed to builtins.listToAttrs"); - Bindings::iterator j = getAttr( - state, - "listToAttrs", - state.sName, - v2->attrs, - pos - ); + Bindings::iterator j = getAttr(state, state.sName, v2->attrs, "in a {name=...; value=...;} pair"); - auto name = state.forceStringNoCtx(*j->value, *j->pos); + auto name = state.forceStringNoCtx(*j->value, j->pos, "while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs"); - Symbol sym = state.symbols.create(name); + auto sym = state.symbols.create(name); if (seen.insert(sym).second) { - Bindings::iterator j2 = getAttr( - state, - "listToAttrs", - state.sValue, - v2->attrs, - pos - ); + Bindings::iterator j2 = getAttr(state, state.sValue, v2->attrs, "in a {name=...; value=...;} pair"); attrs.insert(sym, j2->value, j2->pos); } } @@ -2349,12 +2506,18 @@ Construct a set from a list specifying the names and values of each attribute. Each element of the list should be a set consisting of a string-valued attribute `name` specifying the name of the attribute, - and an attribute `value` specifying its value. Example: + and an attribute `value` specifying its value. + + In case of duplicate occurrences of the same name, the first + takes precedence. + + Example: ```nix builtins.listToAttrs [ { name = "foo"; value = 123; } { name = "bar"; value = 456; } + { name = "bar"; value = 420; } ] ``` @@ -2367,17 +2530,67 @@ .fun = prim_listToAttrs, }); -static void prim_intersectAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_intersectAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); - state.forceAttrs(*args[1], pos); + state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.intersectAttrs"); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.intersectAttrs"); - auto attrs = state.buildBindings(std::min(args[0]->attrs->size(), args[1]->attrs->size())); + Bindings &left = *args[0]->attrs; + Bindings &right = *args[1]->attrs; - for (auto & i : *args[0]->attrs) { - Bindings::iterator j = args[1]->attrs->find(i.name); - if (j != args[1]->attrs->end()) - attrs.insert(*j); + auto attrs = state.buildBindings(std::min(left.size(), right.size())); + + // The current implementation has good asymptotic complexity and is reasonably + // simple. Further optimization may be possible, but does not seem productive, + // considering the state of eval performance in 2022. + // + // I have looked for reusable and/or standard solutions and these are my + // findings: + // + // STL + // === + // std::set_intersection is not suitable, as it only performs a simultaneous + // linear scan; not taking advantage of random access. This is O(n + m), so + // linear in the largest set, which is not acceptable for callPackage in Nixpkgs. + // + // Simultaneous scan, with alternating simple binary search + // === + // One alternative algorithm scans the attrsets simultaneously, jumping + // forward using `lower_bound` in case of inequality. This should perform + // well on very similar sets, having a local and predictable access pattern. + // On dissimilar sets, it seems to need more comparisons than the current + // algorithm, as few consecutive attrs match. `lower_bound` could take + // advantage of the decreasing remaining search space, but this causes + // the medians to move, which can mean that they don't stay in the cache + // like they would with the current naive `find`. + // + // Double binary search + // === + // The optimal algorithm may be "Double binary search", which doesn't + // scan at all, but rather divides both sets simultaneously. + // See "Fast Intersection Algorithms for Sorted Sequences" by Baeza-Yates et al. + // https://cs.uwaterloo.ca/~ajsaling/papers/intersection_alg_app10.pdf + // The only downsides I can think of are not having a linear access pattern + // for similar sets, and having to maintain a more intricate algorithm. + // + // Adaptive + // === + // Finally one could run try a simultaneous scan, count misses and fall back + // to double binary search when the counter hit some threshold and/or ratio. + + if (left.size() < right.size()) { + for (auto & l : left) { + Bindings::iterator r = right.find(l.name); + if (r != right.end()) + attrs.insert(*r); + } + } + else { + for (auto & r : right) { + Bindings::iterator l = left.find(r.name); + if (l != left.end()) + attrs.insert(r); + } } v.mkAttrs(attrs.alreadySorted()); @@ -2387,22 +2600,24 @@ .name = "__intersectAttrs", .args = {"e1", "e2"}, .doc = R"( - Return a set consisting of the attributes in the set *e2* that also - exist in the set *e1*. + Return a set consisting of the attributes in the set *e2* which have the + same name as some attribute in *e1*. + + Performs in O(*n* log *m*) where *n* is the size of the smaller set and *m* the larger set's size. )", .fun = prim_intersectAttrs, }); -static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - Symbol attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos)); - state.forceList(*args[1], pos); + auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); Value * res[args[1]->listSize()]; unsigned int found = 0; for (auto v2 : args[1]->listItems()) { - state.forceAttrs(*v2, pos); + state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs"); Bindings::iterator i = v2->attrs->find(attrName); if (i != v2->attrs->end()) res[found++] = i->value; @@ -2430,7 +2645,7 @@ .fun = prim_catAttrs, }); -static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_functionArgs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); if (args[0]->isPrimOpApp() || args[0]->isPrimOp()) { @@ -2438,10 +2653,10 @@ return; } if (!args[0]->isLambda()) - throw TypeError({ + state.debugThrowLastTrace(TypeError({ .msg = hintfmt("'functionArgs' requires a function"), - .errPos = pos - }); + .errPos = state.positions[pos] + })); if (!args[0]->lambda.fun->hasFormals()) { v.mkAttrs(&state.emptyBindings); @@ -2451,7 +2666,7 @@ auto attrs = state.buildBindings(args[0]->lambda.fun->formals->formals.size()); for (auto & i : args[0]->lambda.fun->formals->formals) // !!! should optimise booleans (allocate only once) - attrs.alloc(i.name, ptr(&i.pos)).mkBool(i.def); + attrs.alloc(i.name, i.pos).mkBool(i.def); v.mkAttrs(attrs); } @@ -2473,16 +2688,16 @@ }); /* */ -static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_mapAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[1], pos); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.mapAttrs"); auto attrs = state.buildBindings(args[1]->attrs->size()); for (auto & i : *args[1]->attrs) { Value * vName = state.allocValue(); Value * vFun2 = state.allocValue(); - vName->mkString(i.name); + vName->mkString(state.symbols[i.name]); vFun2->mkApp(args[0], vName); attrs.alloc(i.name).mkApp(vFun2, i.value); } @@ -2505,7 +2720,7 @@ .fun = prim_mapAttrs, }); -static void prim_zipAttrsWith(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value * * args, Value & v) { // we will first count how many values are present for each given key. // we then allocate a single attrset and pre-populate it with lists of @@ -2516,21 +2731,16 @@ std::map> attrsSeen; - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.zipAttrsWith"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.zipAttrsWith"); const auto listSize = args[1]->listSize(); const auto listElems = args[1]->listElems(); for (unsigned int n = 0; n < listSize; ++n) { Value * vElem = listElems[n]; - try { - state.forceAttrs(*vElem, noPos); - for (auto & attr : *vElem->attrs) - attrsSeen[attr.name].first++; - } catch (TypeError & e) { - e.addTrace(pos, hintfmt("while invoking '%s'", "zipAttrsWith")); - throw; - } + state.forceAttrs(*vElem, noPos, "while evaluating a value of the list passed as second argument to builtins.zipAttrsWith"); + for (auto & attr : *vElem->attrs) + attrsSeen[attr.name].first++; } auto attrs = state.buildBindings(attrsSeen.size()); @@ -2549,7 +2759,7 @@ for (auto & attr : *v.attrs) { auto name = state.allocValue(); - name->mkString(attr.name); + name->mkString(state.symbols[attr.name]); auto call1 = state.allocValue(); call1->mkApp(args[0], name); auto call2 = state.allocValue(); @@ -2597,7 +2807,7 @@ /* Determine whether the argument is a list. */ -static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_isList(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); v.mkBool(args[0]->type() == nList); @@ -2612,22 +2822,22 @@ .fun = prim_isList, }); -static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Value & v) +static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Value & v) { - state.forceList(list, pos); + state.forceList(list, pos, "while evaluating the first argument passed to builtins.elemAt"); if (n < 0 || (unsigned int) n >= list.listSize()) - throw Error({ + state.debugThrowLastTrace(Error({ .msg = hintfmt("list index %1% is out of bounds", n), - .errPos = pos - }); + .errPos = state.positions[pos] + })); state.forceValue(*list.listElems()[n], pos); v = *list.listElems()[n]; } /* Return the n-1'th element of a list. */ -static void prim_elemAt(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - elemAt(state, pos, *args[0], state.forceInt(*args[1], pos), v); + elemAt(state, pos, *args[0], state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt"), v); } static RegisterPrimOp primop_elemAt({ @@ -2641,7 +2851,7 @@ }); /* Return the first element of a list. */ -static void prim_head(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_head(EvalState & state, const PosIdx pos, Value * * args, Value & v) { elemAt(state, pos, *args[0], 0, v); } @@ -2660,14 +2870,14 @@ /* Return a list consisting of everything but the first element of a list. Warning: this function takes O(n) time, so you probably don't want to use it! */ -static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_tail(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.tail"); if (args[0]->listSize() == 0) - throw Error({ + state.debugThrowLastTrace(Error({ .msg = hintfmt("'tail' called on an empty list"), - .errPos = pos - }); + .errPos = state.positions[pos] + })); state.mkList(v, args[0]->listSize() - 1); for (unsigned int n = 0; n < v.listSize(); ++n) @@ -2691,12 +2901,18 @@ }); /* Apply a function to every element of a list. */ -static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_map(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[1], pos); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.map"); - state.mkList(v, args[1]->listSize()); + if (args[1]->listSize() == 0) { + v = *args[1]; + return; + } + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.map"); + + state.mkList(v, args[1]->listSize()); for (unsigned int n = 0; n < v.listSize(); ++n) (v.listElems()[n] = state.allocValue())->mkApp( args[0], args[1]->listElems()[n]); @@ -2721,10 +2937,16 @@ /* Filter a list using a predicate; that is, return a list containing every element from the list for which the predicate function returns true. */ -static void prim_filter(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.filter"); + + if (args[1]->listSize() == 0) { + v = *args[1]; + return; + } + + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); // FIXME: putting this on the stack is risky. Value * vs[args[1]->listSize()]; @@ -2734,7 +2956,7 @@ for (unsigned int n = 0; n < args[1]->listSize(); ++n) { Value res; state.callFunction(*args[0], *args[1]->listElems()[n], res, noPos); - if (state.forceBool(res, pos)) + if (state.forceBool(res, pos, "while evaluating the return value of the filtering function passed to builtins.filter")) vs[k++] = args[1]->listElems()[n]; else same = false; @@ -2759,12 +2981,12 @@ }); /* Return true if a list contains a given element. */ -static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_elem(EvalState & state, const PosIdx pos, Value * * args, Value & v) { bool res = false; - state.forceList(*args[1], pos); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.elem"); for (auto elem : args[1]->listItems()) - if (state.eqValues(*args[0], *elem)) { + if (state.eqValues(*args[0], *elem, pos, "while searching for the presence of the given element in the list")) { res = true; break; } @@ -2782,10 +3004,10 @@ }); /* Concatenate a list of lists. */ -static void prim_concatLists(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_concatLists(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); - state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.concatLists"); + state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos, "while evaluating a value of the list passed to builtins.concatLists"); } static RegisterPrimOp primop_concatLists({ @@ -2798,9 +3020,9 @@ }); /* Return the length of a list. This is an O(1) time operation. */ -static void prim_length(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_length(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.length"); v.mkInt(args[0]->listSize()); } @@ -2815,10 +3037,10 @@ /* Reduce a list by applying a binary operator, from left to right. The operator is applied strictly. */ -static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_foldlStrict(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[2], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.foldlStrict"); + state.forceList(*args[2], pos, "while evaluating the third argument passed to builtins.foldlStrict"); if (args[2]->listSize()) { Value * vCur = args[1]; @@ -2841,22 +3063,22 @@ .doc = R"( Reduce a list by applying a binary operator, from left to right, e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. The operator is applied strictly, i.e., its arguments are - evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]` - evaluates to 6. + ...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. + The return value of each application of `op` is evaluated immediately, + even for intermediate values. )", .fun = prim_foldlStrict, }); -static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * args, Value & v) +static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, std::string("while evaluating the first argument passed to builtins.") + (any ? "any" : "all")); + state.forceList(*args[1], pos, std::string("while evaluating the second argument passed to builtins.") + (any ? "any" : "all")); Value vTmp; for (auto elem : args[1]->listItems()) { state.callFunction(*args[0], *elem, vTmp, pos); - bool res = state.forceBool(vTmp, pos); + bool res = state.forceBool(vTmp, pos, std::string("while evaluating the return value of the function passed to builtins.") + (any ? "any" : "all")); if (res == any) { v.mkBool(any); return; @@ -2867,7 +3089,7 @@ } -static void prim_any(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_any(EvalState & state, const PosIdx pos, Value * * args, Value & v) { anyOrAll(true, state, pos, args, v); } @@ -2882,7 +3104,7 @@ .fun = prim_any, }); -static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_all(EvalState & state, const PosIdx pos, Value * * args, Value & v) { anyOrAll(false, state, pos, args, v); } @@ -2897,18 +3119,18 @@ .fun = prim_all, }); -static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto len = state.forceInt(*args[1], pos); + auto len = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList"); if (len < 0) - throw EvalError({ - .msg = hintfmt("cannot create list of size %1%", len), - .errPos = pos - }); + state.error("cannot create list of size %1%", len).debugThrow(); - state.mkList(v, len); + // More strict than striclty (!) necessary, but acceptable + // as evaluating map without accessing any values makes little sense. + state.forceFunction(*args[0], noPos, "while evaluating the first argument passed to builtins.genList"); + state.mkList(v, len); for (unsigned int n = 0; n < (unsigned int) len; ++n) { auto arg = state.allocValue(); arg->mkInt(n); @@ -2932,15 +3154,21 @@ .fun = prim_genList, }); -static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v); +static void prim_lessThan(EvalState & state, const PosIdx pos, Value * * args, Value & v); -static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_sort(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.sort"); auto len = args[1]->listSize(); + if (len == 0) { + v = *args[1]; + return; + } + + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.sort"); + state.mkList(v, len); for (unsigned int n = 0; n < len; ++n) { state.forceValue(*args[1]->listElems()[n], pos); @@ -2950,13 +3178,15 @@ auto comparator = [&](Value * a, Value * b) { /* Optimization: if the comparator is lessThan, bypass callFunction. */ + /* TODO: (layus) this is absurd. An optimisation like this + should be outside the lambda creation */ if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) - return CompareValues(state)(a, b); + return CompareValues(state, noPos, "while evaluating the ordering function passed to builtins.sort")(a, b); Value * vs[] = {a, b}; Value vBool; - state.callFunction(*args[0], 2, vs, vBool, pos); - return state.forceBool(vBool, pos); + state.callFunction(*args[0], 2, vs, vBool, noPos); + return state.forceBool(vBool, pos, "while evaluating the return value of the sorting function passed to builtins.sort"); }; /* FIXME: std::sort can segfault if the comparator is not a strict @@ -2986,10 +3216,10 @@ .fun = prim_sort, }); -static void prim_partition(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_partition(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.partition"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.partition"); auto len = args[1]->listSize(); @@ -3000,7 +3230,7 @@ state.forceValue(*vElem, pos); Value res; state.callFunction(*args[0], *vElem, res, pos); - if (state.forceBool(res, pos)) + if (state.forceBool(res, pos, "while evaluating the return value of the partition function passed to builtins.partition")) right.push_back(vElem); else wrong.push_back(vElem); @@ -3046,18 +3276,18 @@ .fun = prim_partition, }); -static void prim_groupBy(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_groupBy(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.groupBy"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.groupBy"); ValueVectorMap attrs; for (auto vElem : args[1]->listItems()) { Value res; state.callFunction(*args[0], *vElem, res, pos); - auto name = state.forceStringNoCtx(res, pos); - Symbol sym = state.symbols.create(name); + auto name = state.forceStringNoCtx(res, pos, "while evaluating the return value of the grouping function passed to builtins.groupBy"); + auto sym = state.symbols.create(name); auto vector = attrs.try_emplace(sym, ValueVector()).first; vector->second.push_back(vElem); } @@ -3098,10 +3328,10 @@ .fun = prim_groupBy, }); -static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.concatMap"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); Value lists[nrLists]; @@ -3110,12 +3340,7 @@ for (unsigned int n = 0; n < nrLists; ++n) { Value * vElem = args[1]->listElems()[n]; state.callFunction(*args[0], *vElem, lists[n], pos); - try { - state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos))); - } catch (TypeError &e) { - e.addTrace(pos, hintfmt("while invoking '%s'", "concatMap")); - throw; - } + state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to buitlins.concatMap"); len += lists[n].listSize(); } @@ -3145,14 +3370,16 @@ *************************************************************/ -static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_add(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - v.mkFloat(state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the addition") + + state.forceFloat(*args[1], pos, "while evaluating the second argument of the addition")); else - v.mkInt(state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); + v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the addition") + + state.forceInt(*args[1], pos, "while evaluating the second argument of the addition")); } static RegisterPrimOp primop_add({ @@ -3164,14 +3391,16 @@ .fun = prim_add, }); -static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_sub(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - v.mkFloat(state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the subtraction") + - state.forceFloat(*args[1], pos, "while evaluating the second argument of the subtraction")); else - v.mkInt(state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); + v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction") + - state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction")); } static RegisterPrimOp primop_sub({ @@ -3183,14 +3412,16 @@ .fun = prim_sub, }); -static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_mul(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - v.mkFloat(state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first of the multiplication") + * state.forceFloat(*args[1], pos, "while evaluating the second argument of the multiplication")); else - v.mkInt(state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); + v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication") + * state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication")); } static RegisterPrimOp primop_mul({ @@ -3202,29 +3433,29 @@ .fun = prim_mul, }); -static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - NixFloat f2 = state.forceFloat(*args[1], pos); + NixFloat f2 = state.forceFloat(*args[1], pos, "while evaluating the second operand of the division"); if (f2 == 0) - throw EvalError({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("division by zero"), - .errPos = pos - }); + .errPos = state.positions[pos] + })); if (args[0]->type() == nFloat || args[1]->type() == nFloat) { - v.mkFloat(state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first operand of the division") / f2); } else { - NixInt i1 = state.forceInt(*args[0], pos); - NixInt i2 = state.forceInt(*args[1], pos); + NixInt i1 = state.forceInt(*args[0], pos, "while evaluating the first operand of the division"); + NixInt i2 = state.forceInt(*args[1], pos, "while evaluating the second operand of the division"); /* Avoid division overflow as it might raise SIGFPE. */ if (i1 == std::numeric_limits::min() && i2 == -1) - throw EvalError({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("overflow in integer division"), - .errPos = pos - }); + .errPos = state.positions[pos] + })); v.mkInt(i1 / i2); } @@ -3239,9 +3470,10 @@ .fun = prim_div, }); -static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_bitAnd(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd") + & state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd")); } static RegisterPrimOp primop_bitAnd({ @@ -3253,9 +3485,10 @@ .fun = prim_bitAnd, }); -static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_bitOr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr") + | state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr")); } static RegisterPrimOp primop_bitOr({ @@ -3267,9 +3500,10 @@ .fun = prim_bitOr, }); -static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_bitXor(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor") + ^ state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor")); } static RegisterPrimOp primop_bitXor({ @@ -3281,11 +3515,12 @@ .fun = prim_bitXor, }); -static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_lessThan(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - CompareValues comp{state}; + // pos is exact here, no need for a message. + CompareValues comp(state, noPos, ""); v.mkBool(comp(args[0], args[1])); } @@ -3309,10 +3544,12 @@ /* Convert the argument to a string. Paths are *not* copied to the store, so `toString /foo/bar' yields `"/foo/bar"', not `"/nix/store/whatever..."'. */ -static void prim_toString(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_toString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto s = state.coerceToString(pos, *args[0], context, true, false); + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to builtins.toString", + true, false); v.mkString(*s, context); } @@ -3344,18 +3581,18 @@ at character position `min(start, stringLength str)' inclusive and ending at `min(start + len, stringLength str)'. `start' must be non-negative. */ -static void prim_substring(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - int start = state.forceInt(*args[0], pos); - int len = state.forceInt(*args[1], pos); - PathSet context; - auto s = state.coerceToString(pos, *args[2], context); + int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring"); + int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring"); + NixStringContext context; + auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring"); if (start < 0) - throw EvalError({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("negative start position in 'substring'"), - .errPos = pos - }); + .errPos = state.positions[pos] + })); v.mkString((unsigned int) start >= s->size() ? "" : s->substr(start, len), context); } @@ -3380,10 +3617,10 @@ .fun = prim_substring, }); -static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto s = state.coerceToString(pos, *args[0], context); + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength"); v.mkInt(s->size()); } @@ -3398,18 +3635,18 @@ }); /* Return the cryptographic hash of a string in base-16. */ -static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto type = state.forceStringNoCtx(*args[0], pos); + auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString"); std::optional ht = parseHashType(type); if (!ht) - throw Error({ + state.debugThrowLastTrace(Error({ .msg = hintfmt("unknown hash type '%1%'", type), - .errPos = pos - }); + .errPos = state.positions[pos] + })); - PathSet context; // discarded - auto s = state.forceString(*args[1], context, pos); + NixStringContext context; // discarded + auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); v.mkString(hashString(*ht, s).to_string(Base16, false)); } @@ -3446,16 +3683,16 @@ return std::make_shared(); } -void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) +void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto re = state.forceStringNoCtx(*args[0], pos); + auto re = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.match"); try { auto regex = state.regexCache->get(re); - PathSet context; - const auto str = state.forceString(*args[1], context, pos); + NixStringContext context; + const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); std::cmatch match; if (!std::regex_match(str.begin(), str.end(), match, regex)) { @@ -3473,19 +3710,18 @@ (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str()); } - } catch (std::regex_error &e) { + } catch (std::regex_error & e) { if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - throw EvalError({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), - .errPos = pos - }); - } else { - throw EvalError({ + .errPos = state.positions[pos] + })); + } else + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("invalid regular expression '%s'", re), - .errPos = pos - }); - } + .errPos = state.positions[pos] + })); } } @@ -3520,23 +3756,23 @@ builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " ``` - Evaluates to `[ "foo" ]`. + Evaluates to `[ "FOO" ]`. )s", .fun = prim_match, }); /* Split a string with a regular expression, and return a list of the non-matching parts interleaved by the lists of the matching groups. */ -void prim_split(EvalState & state, const Pos & pos, Value * * args, Value & v) +void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto re = state.forceStringNoCtx(*args[0], pos); + auto re = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.split"); try { auto regex = state.regexCache->get(re); - PathSet context; - const auto str = state.forceString(*args[1], context, pos); + NixStringContext context; + const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); auto begin = std::cregex_iterator(str.begin(), str.end(), regex); auto end = std::cregex_iterator(); @@ -3578,19 +3814,18 @@ assert(idx == 2 * len + 1); - } catch (std::regex_error &e) { + } catch (std::regex_error & e) { if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - throw EvalError({ + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), - .errPos = pos - }); - } else { - throw EvalError({ + .errPos = state.positions[pos] + })); + } else + state.debugThrowLastTrace(EvalError({ .msg = hintfmt("invalid regular expression '%s'", re), - .errPos = pos - }); - } + .errPos = state.positions[pos] + })); } } @@ -3623,7 +3858,7 @@ Evaluates to `[ "" [ "a" null ] "b" [ null "c" ] "" ]`. ```nix - builtins.split "([[:upper:]]+)" " FOO " + builtins.split "([[:upper:]]+)" " FOO " ``` Evaluates to `[ " " [ "FOO" ] " " ]`. @@ -3631,12 +3866,12 @@ .fun = prim_split, }); -static void prim_concatStringsSep(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; - auto sep = state.forceString(*args[0], context, pos); - state.forceList(*args[1], pos); + auto sep = state.forceString(*args[0], context, pos, "while evaluating the first argument (the separator string) passed to builtins.concatStringsSep"); + state.forceList(*args[1], pos, "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep"); std::string res; res.reserve((args[1]->listSize() + 32) * sep.size()); @@ -3644,7 +3879,7 @@ for (auto elem : args[1]->listItems()) { if (first) first = false; else res += sep; - res += *state.coerceToString(pos, *elem, context); + res += *state.coerceToString(pos, *elem, context, "while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep"); } v.mkString(res, context); @@ -3661,31 +3896,23 @@ .fun = prim_concatStringsSep, }); -static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); - state.forceList(*args[1], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.replaceStrings"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.replaceStrings"); if (args[0]->listSize() != args[1]->listSize()) - throw EvalError({ - .msg = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), - .errPos = pos - }); + state.error("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths").atPos(pos).debugThrow(); std::vector from; from.reserve(args[0]->listSize()); for (auto elem : args[0]->listItems()) - from.emplace_back(state.forceString(*elem, pos)); + from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings")); - std::vector> to; - to.reserve(args[1]->listSize()); - for (auto elem : args[1]->listItems()) { - PathSet ctx; - auto s = state.forceString(*elem, ctx, pos); - to.emplace_back(s, std::move(ctx)); - } + std::unordered_map cache; + auto to = args[1]->listItems(); - PathSet context; - auto s = state.forceString(*args[2], context, pos); + NixStringContext context; + auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings"); std::string res; // Loops one past last character to handle the case where 'from' contains an empty string. @@ -3693,10 +3920,19 @@ bool found = false; auto i = from.begin(); auto j = to.begin(); - for (; i != from.end(); ++i, ++j) + size_t j_index = 0; + for (; i != from.end(); ++i, ++j, ++j_index) if (s.compare(p, i->size(), *i) == 0) { found = true; - res += j->first; + auto v = cache.find(j_index); + if (v == cache.end()) { + NixStringContext ctx; + auto ts = state.forceString(**j, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings"); + v = (cache.emplace(j_index, ts)).first; + for (auto& path : ctx) + context.insert(path); + } + res += v->second; if (i->empty()) { if (p < s.size()) res += s[p]; @@ -3704,9 +3940,6 @@ } else { p += i->size(); } - for (auto& path : j->second) - context.insert(path); - j->second.clear(); break; } if (!found) { @@ -3724,7 +3957,11 @@ .args = {"from", "to", "s"}, .doc = R"( Given string *s*, replace every occurrence of the strings in *from* - with the corresponding string in *to*. For example, + with the corresponding string in *to*. + + The argument *to* is lazy, that is, it is only evaluated when its corresponding pattern in *from* is matched in the string *s* + + Example: ```nix builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" @@ -3741,9 +3978,9 @@ *************************************************************/ -static void prim_parseDrvName(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_parseDrvName(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto name = state.forceStringNoCtx(*args[0], pos); + auto name = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.parseDrvName"); DrvName parsed(name); auto attrs = state.buildBindings(2); attrs.alloc(state.sName).mkString(parsed.name); @@ -3756,8 +3993,8 @@ .args = {"s"}, .doc = R"( Split the string *s* into a package name and version. The package - name is everything up to but not including the first dash followed - by a digit, and the version is everything following that dash. The + name is everything up to but not including the first dash not followed + by a letter, and the version is everything following that dash. The result is returned in a set `{ name, version }`. Thus, `builtins.parseDrvName "nix-0.12pre12876"` returns `{ name = "nix"; version = "0.12pre12876"; }`. @@ -3765,10 +4002,10 @@ .fun = prim_parseDrvName, }); -static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_compareVersions(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto version1 = state.forceStringNoCtx(*args[0], pos); - auto version2 = state.forceStringNoCtx(*args[1], pos); + auto version1 = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.compareVersions"); + auto version2 = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.compareVersions"); v.mkInt(compareVersions(version1, version2)); } @@ -3785,9 +4022,9 @@ .fun = prim_compareVersions, }); -static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_splitVersion(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto version = state.forceStringNoCtx(*args[0], pos); + auto version = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.splitVersion"); auto iter = version.cbegin(); Strings components; while (iter != version.cend()) { @@ -3887,6 +4124,18 @@ addPrimOp("__exec", 1, prim_exec); } + addPrimOp({ + .fun = evalSettings.traceVerbose ? prim_trace : prim_second, + .arity = 2, + .name = "__traceVerbose", + .args = { "e1", "e2" }, + .doc = R"( + Evaluate *e1* and print its abstract syntax representation on standard + error if `--trace-verbose` is enabled. Then return *e2*. This function + is useful for debugging. + )", + }); + /* Add a value containing the current Nix expression search path. */ mkList(v, searchPath.size()); int n = 0; @@ -3901,12 +4150,12 @@ if (RegisterPrimOp::primOps) for (auto & primOp : *RegisterPrimOp::primOps) if (!primOp.experimentalFeature - || settings.isExperimentalFeatureEnabled(*primOp.experimentalFeature)) + || experimentalFeatureSettings.isEnabled(*primOp.experimentalFeature)) { addPrimOp({ .fun = primOp.fun, .arity = std::max(primOp.args.size(), primOp.arity), - .name = symbols.create(primOp.name), + .name = primOp.name, .args = primOp.args, .doc = primOp.doc, }); @@ -3914,7 +4163,6 @@ /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ - sDerivationNix = symbols.create("//builtin/derivation.nix"); auto vDerivation = allocValue(); addConstant("derivation", vDerivation); @@ -3922,7 +4170,7 @@ because attribute lookups expect it to be sorted. */ baseEnv.values[0]->attrs->sort(); - staticBaseEnv.sort(); + staticBaseEnv->sort(); /* Note: we have to initialize the 'derivation' constant *after* building baseEnv/staticBaseEnv because it uses 'builtins'. */ @@ -3931,7 +4179,7 @@ // the parser needs two NUL bytes as terminators; one of them // is implied by being a C string. "\0"; - eval(parse(code, sizeof(code), foFile, sDerivationNix, "/", staticBaseEnv), *vDerivation); + eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation); } diff -Nru nix-2.8.0/src/libexpr/primops.hh nix-2.16.1+dfsg/src/libexpr/primops.hh --- nix-2.8.0/src/libexpr/primops.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/primops.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" @@ -22,9 +23,11 @@ typedef std::vector PrimOps; static PrimOps * primOps; - /* You can register a constant by passing an arity of 0. fun - will get called during EvalState initialization, so there - may be primops not yet added and builtins is not yet sorted. */ + /** + * You can register a constant by passing an arity of 0. fun + * will get called during EvalState initialization, so there + * may be primops not yet added and builtins is not yet sorted. + */ RegisterPrimOp( std::string name, size_t arity, @@ -37,10 +40,14 @@ may wish to use them in limited contexts without globally enabling them. */ -/* Load a ValueInitializer from a DSO and return whatever it initializes */ -void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v); - -/* Execute a program and parse its output */ -void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v); +/** + * Load a ValueInitializer from a DSO and return whatever it initializes + */ +void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Value & v); + +/** + * Execute a program and parse its output + */ +void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v); } diff -Nru nix-2.8.0/src/libexpr/print.cc nix-2.16.1+dfsg/src/libexpr/print.cc --- nix-2.8.0/src/libexpr/print.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/print.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,94 @@ +#include "print.hh" +#include + +namespace nix { + +std::ostream & +printLiteralString(std::ostream & str, const std::string_view string) +{ + str << "\""; + for (auto i = string.begin(); i != string.end(); ++i) { + if (*i == '\"' || *i == '\\') str << "\\" << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; + else str << *i; + } + str << "\""; + return str; +} + +std::ostream & +printLiteralBool(std::ostream & str, bool boolean) +{ + str << (boolean ? "true" : "false"); + return str; +} + +// Returns `true' is a string is a reserved keyword which requires quotation +// when printing attribute set field names. +// +// This list should generally be kept in sync with `./lexer.l'. +// You can test if a keyword needs to be added by running: +// $ nix eval --expr '{ = 1; }' +// For example `or' doesn't need to be quoted. +bool isReservedKeyword(const std::string_view str) +{ + static const std::unordered_set reservedKeywords = { + "if", "then", "else", "assert", "with", "let", "in", "rec", "inherit" + }; + return reservedKeywords.contains(str); +} + +std::ostream & +printIdentifier(std::ostream & str, std::string_view s) { + if (s.empty()) + str << "\"\""; + else if (isReservedKeyword(s)) + str << '"' << s << '"'; + else { + char c = s[0]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { + printLiteralString(str, s); + return str; + } + for (auto c : s) + if (!((c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '_' || c == '\'' || c == '-')) { + printLiteralString(str, s); + return str; + } + str << s; + } + return str; +} + +static bool isVarName(std::string_view s) +{ + if (s.size() == 0) return false; + if (isReservedKeyword(s)) return false; + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; + for (auto & i : s) + if (!((i >= 'a' && i <= 'z') || + (i >= 'A' && i <= 'Z') || + (i >= '0' && i <= '9') || + i == '_' || i == '-' || i == '\'')) + return false; + return true; +} + +std::ostream & +printAttributeName(std::ostream & str, std::string_view name) { + if (isVarName(name)) + str << name; + else + printLiteralString(str, name); + return str; +} + + +} diff -Nru nix-2.8.0/src/libexpr/print.hh nix-2.16.1+dfsg/src/libexpr/print.hh --- nix-2.8.0/src/libexpr/print.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/print.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,54 @@ +#pragma once +/** + * @file + * @brief Common printing functions for the Nix language + * + * While most types come with their own methods for printing, they share some + * functions that are placed here. + */ + +#include + +namespace nix { + /** + * Print a string as a Nix string literal. + * + * Quotes and fairly minimal escaping are added. + * + * @param s The logical string + */ + std::ostream & printLiteralString(std::ostream & o, std::string_view s); + inline std::ostream & printLiteralString(std::ostream & o, const char * s) { + return printLiteralString(o, std::string_view(s)); + } + inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) { + return printLiteralString(o, std::string_view(s)); + } + + /** Print `true` or `false`. */ + std::ostream & printLiteralBool(std::ostream & o, bool b); + + /** + * Print a string as an attribute name in the Nix expression language syntax. + * + * Prints a quoted string if necessary. + */ + std::ostream & printAttributeName(std::ostream & o, std::string_view s); + + /** + * Returns `true' is a string is a reserved keyword which requires quotation + * when printing attribute set field names. + */ + bool isReservedKeyword(const std::string_view str); + + /** + * Print a string as an identifier in the Nix expression language syntax. + * + * FIXME: "identifier" is ambiguous. Identifiers do not have a single + * textual representation. They can be used in variable references, + * let bindings, left-hand sides or attribute names in a select + * expression, or something else entirely, like JSON. Use one of the + * `print*` functions instead. + */ + std::ostream & printIdentifier(std::ostream & o, std::string_view s); +} diff -Nru nix-2.8.0/src/libexpr/symbol-table.hh nix-2.16.1+dfsg/src/libexpr/symbol-table.hh --- nix-2.8.0/src/libexpr/symbol-table.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/symbol-table.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,48 +1,33 @@ #pragma once +///@file #include #include #include #include "types.hh" +#include "chunked-vector.hh" namespace nix { -/* Symbol table used by the parser and evaluator to represent and look - up identifiers and attributes efficiently. SymbolTable::create() - converts a string into a symbol. Symbols have the property that - they can be compared efficiently (using a pointer equality test), - because the symbol table stores only one copy of each string. */ - -class Symbol +/** + * This class mainly exists to give us an operator<< for ostreams. We could also + * return plain strings from SymbolTable, but then we'd have to wrap every + * instance of a symbol that is fmt()ed, which is inconvenient and error-prone. + */ +class SymbolStr { -private: - const std::string * s; // pointer into SymbolTable - Symbol(const std::string * s) : s(s) { }; friend class SymbolTable; -public: - Symbol() : s(0) { }; +private: + const std::string * s; - bool operator == (const Symbol & s2) const - { - return s == s2.s; - } + explicit SymbolStr(const std::string & symbol): s(&symbol) {} - // FIXME: remove +public: bool operator == (std::string_view s2) const { - return s->compare(s2) == 0; - } - - bool operator != (const Symbol & s2) const - { - return s != s2.s; - } - - bool operator < (const Symbol & s2) const - { - return s < s2.s; + return *s == s2; } operator const std::string & () const @@ -55,51 +40,90 @@ return *s; } - bool set() const - { - return s; - } + friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol); +}; - bool empty() const - { - return s->empty(); - } +/** + * Symbols have the property that they can be compared efficiently + * (using an equality test), because the symbol table stores only one + * copy of each string. + */ +class Symbol +{ + friend class SymbolTable; + +private: + uint32_t id; + + explicit Symbol(uint32_t id): id(id) {} + +public: + Symbol() : id(0) {} - friend std::ostream & operator << (std::ostream & str, const Symbol & sym); + explicit operator bool() const { return id > 0; } + + bool operator<(const Symbol other) const { return id < other.id; } + bool operator==(const Symbol other) const { return id == other.id; } + bool operator!=(const Symbol other) const { return id != other.id; } }; +/** + * Symbol table used by the parser and evaluator to represent and look + * up identifiers and attributes efficiently. + */ class SymbolTable { private: - std::unordered_map symbols; - std::list store; + std::unordered_map> symbols; + ChunkedVector store{16}; public: + + /** + * converts a string into a symbol. + */ Symbol create(std::string_view s) { // Most symbols are looked up more than once, so we trade off insertion performance // for lookup performance. // TODO: could probably be done more efficiently with transparent Hash and Equals // on the original implementation using unordered_set + // FIXME: make this thread-safe. auto it = symbols.find(s); - if (it != symbols.end()) return it->second; + if (it != symbols.end()) return Symbol(it->second.second + 1); + + const auto & [rawSym, idx] = store.add(std::string(s)); + symbols.emplace(rawSym, std::make_pair(&rawSym, idx)); + return Symbol(idx + 1); + } - auto & rawSym = store.emplace_back(s); - return symbols.emplace(rawSym, Symbol(&rawSym)).first->second; + std::vector resolve(const std::vector & symbols) const + { + std::vector result; + result.reserve(symbols.size()); + for (auto sym : symbols) + result.push_back((*this)[sym]); + return result; + } + + SymbolStr operator[](Symbol s) const + { + if (s.id == 0 || s.id > store.size()) + abort(); + return SymbolStr(store[s.id - 1]); } size_t size() const { - return symbols.size(); + return store.size(); } size_t totalSize() const; template - void dump(T callback) + void dump(T callback) const { - for (auto & s : store) - callback(s); + store.forEach(callback); } }; diff -Nru nix-2.8.0/src/libexpr/tests/derived-path.cc nix-2.16.1+dfsg/src/libexpr/tests/derived-path.cc --- nix-2.8.0/src/libexpr/tests/derived-path.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/tests/derived-path.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,65 @@ +#include +#include +#include + +#include "tests/derived-path.hh" +#include "tests/libexpr.hh" + +namespace nix { + +// Testing of trivial expressions +class DerivedPathExpressionTest : public LibExprTest {}; + +// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is +// no a real fixture. +// +// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args +TEST_F(DerivedPathExpressionTest, force_init) +{ +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathExpressionTest, + prop_opaque_path_round_trip, + (const DerivedPath::Opaque & o)) +{ + auto * v = state.allocValue(); + state.mkStorePathString(o.path, *v); + auto d = state.coerceToDerivedPath(noPos, *v, ""); + RC_ASSERT(DerivedPath { o } == d); +} + +// TODO use DerivedPath::Built for parameter once it supports a single output +// path only. + +RC_GTEST_FIXTURE_PROP( + DerivedPathExpressionTest, + prop_built_path_placeholder_round_trip, + (const StorePath & drvPath, const StorePathName & outputName)) +{ + auto * v = state.allocValue(); + state.mkOutputString(*v, drvPath, outputName.name, std::nullopt); + auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); + DerivedPath::Built b { + .drvPath = drvPath, + .outputs = OutputsSpec::Names { outputName.name }, + }; + RC_ASSERT(DerivedPath { b } == d); +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathExpressionTest, + prop_built_path_out_path_round_trip, + (const StorePath & drvPath, const StorePathName & outputName, const StorePath & outPath)) +{ + auto * v = state.allocValue(); + state.mkOutputString(*v, drvPath, outputName.name, outPath); + auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); + DerivedPath::Built b { + .drvPath = drvPath, + .outputs = OutputsSpec::Names { outputName.name }, + }; + RC_ASSERT(DerivedPath { b } == d); +} + +} /* namespace nix */ diff -Nru nix-2.8.0/src/libexpr/tests/error_traces.cc nix-2.16.1+dfsg/src/libexpr/tests/error_traces.cc --- nix-2.8.0/src/libexpr/tests/error_traces.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/tests/error_traces.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,1298 @@ +#include +#include + +#include "tests/libexpr.hh" + +namespace nix { + + using namespace testing; + + // Testing eval of PrimOp's + class ErrorTraceTest : public LibExprTest { }; + + TEST_F(ErrorTraceTest, TraceBuilder) { + ASSERT_THROW( + state.error("Not much").debugThrow(), + EvalError + ); + + ASSERT_THROW( + state.error("Not much").withTrace(noPos, "No more").debugThrow(), + EvalError + ); + + ASSERT_THROW( + try { + try { + state.error("Not much").withTrace(noPos, "No more").debugThrow(); + } catch (Error & e) { + e.addTrace(state.positions[noPos], "Something", ""); + throw; + } + } catch (BaseError & e) { + ASSERT_EQ(PrintToString(e.info().msg), + PrintToString(hintfmt("Not much"))); + auto trace = e.info().traces.rbegin(); + ASSERT_EQ(e.info().traces.size(), 2); + ASSERT_EQ(PrintToString(trace->hint), + PrintToString(hintfmt("No more"))); + trace++; + ASSERT_EQ(PrintToString(trace->hint), + PrintToString(hintfmt("Something"))); + throw; + } + , EvalError + ); + } + + TEST_F(ErrorTraceTest, NestedThrows) { + try { + state.error("Not much").withTrace(noPos, "No more").debugThrow(); + } catch (BaseError & e) { + try { + state.error("Not much more").debugThrow(); + } catch (Error & e2) { + e.addTrace(state.positions[noPos], "Something", ""); + //e2.addTrace(state.positions[noPos], "Something", ""); + ASSERT_TRUE(e.info().traces.size() == 2); + ASSERT_TRUE(e2.info().traces.size() == 0); + ASSERT_FALSE(&e.info() == &e2.info()); + } + } + } + +#define ASSERT_TRACE1(args, type, message) \ + ASSERT_THROW( \ + std::string expr(args); \ + std::string name = expr.substr(0, expr.find(" ")); \ + try { \ + Value v = eval("builtins." args); \ + state.forceValueDeep(v); \ + } catch (BaseError & e) { \ + ASSERT_EQ(PrintToString(e.info().msg), \ + PrintToString(message)); \ + ASSERT_EQ(e.info().traces.size(), 1) << "while testing " args << std::endl << e.what(); \ + auto trace = e.info().traces.rbegin(); \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(hintfmt("while calling the '%s' builtin", name))); \ + throw; \ + } \ + , type \ + ) + +#define ASSERT_TRACE2(args, type, message, context) \ + ASSERT_THROW( \ + std::string expr(args); \ + std::string name = expr.substr(0, expr.find(" ")); \ + try { \ + Value v = eval("builtins." args); \ + state.forceValueDeep(v); \ + } catch (BaseError & e) { \ + ASSERT_EQ(PrintToString(e.info().msg), \ + PrintToString(message)); \ + ASSERT_EQ(e.info().traces.size(), 2) << "while testing " args << std::endl << e.what(); \ + auto trace = e.info().traces.rbegin(); \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(context)); \ + ++trace; \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(hintfmt("while calling the '%s' builtin", name))); \ + throw; \ + } \ + , type \ + ) + + TEST_F(ErrorTraceTest, genericClosure) { + ASSERT_TRACE2("genericClosure 1", + TypeError, + hintfmt("value is %s while a set was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure {}", + TypeError, + hintfmt("attribute '%s' missing", "startSet"), + hintfmt("in the attrset passed as argument to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = 1; }", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = true; }", + TypeError, + hintfmt("value is %s while a function was expected", "a Boolean"), + hintfmt("while evaluating the 'operator' attribute passed as argument to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: true; }", + TypeError, + hintfmt("value is %s while a list was expected", "a Boolean"), + hintfmt("while evaluating the return value of the `operator` passed to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [ true ]; }", + TypeError, + hintfmt("value is %s while a set was expected", "a Boolean"), + hintfmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [ {} ]; }", + TypeError, + hintfmt("attribute '%s' missing", "key"), + hintfmt("in one of the attrsets generated by (or initially passed to) builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [{ key = ''a''; }]; }", + EvalError, + hintfmt("cannot compare %s with %s", "a string", "an integer"), + hintfmt("while comparing the `key` attributes of two genericClosure elements")); + + ASSERT_TRACE2("genericClosure { startSet = [ true ]; operator = item: [{ key = ''a''; }]; }", + TypeError, + hintfmt("value is %s while a set was expected", "a Boolean"), + hintfmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); + + } + + + TEST_F(ErrorTraceTest, replaceStrings) { + ASSERT_TRACE2("replaceStrings 0 0 {}", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.replaceStrings")); + + ASSERT_TRACE2("replaceStrings [] 0 {}", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the second argument passed to builtins.replaceStrings")); + + ASSERT_TRACE1("replaceStrings [ 0 ] [] {}", + EvalError, + hintfmt("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths")); + + ASSERT_TRACE2("replaceStrings [ 1 ] [ \"new\" ] {}", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); + + ASSERT_TRACE2("replaceStrings [ \"oo\" ] [ true ] \"foo\"", + TypeError, + hintfmt("value is %s while a string was expected", "a Boolean"), + hintfmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); + + ASSERT_TRACE2("replaceStrings [ \"old\" ] [ \"new\" ] {}", + TypeError, + hintfmt("value is %s while a string was expected", "a set"), + hintfmt("while evaluating the third argument passed to builtins.replaceStrings")); + + } + + + TEST_F(ErrorTraceTest, scopedImport) { + } + + + TEST_F(ErrorTraceTest, import) { + } + + + TEST_F(ErrorTraceTest, typeOf) { + } + + + TEST_F(ErrorTraceTest, isNull) { + } + + + TEST_F(ErrorTraceTest, isFunction) { + } + + + TEST_F(ErrorTraceTest, isInt) { + } + + + TEST_F(ErrorTraceTest, isFloat) { + } + + + TEST_F(ErrorTraceTest, isString) { + } + + + TEST_F(ErrorTraceTest, isBool) { + } + + + TEST_F(ErrorTraceTest, isPath) { + } + + + TEST_F(ErrorTraceTest, break) { + } + + + TEST_F(ErrorTraceTest, abort) { + } + + + TEST_F(ErrorTraceTest, throw) { + } + + + TEST_F(ErrorTraceTest, addErrorContext) { + } + + + TEST_F(ErrorTraceTest, ceil) { + ASSERT_TRACE2("ceil \"foo\"", + TypeError, + hintfmt("value is %s while a float was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.ceil")); + + } + + + TEST_F(ErrorTraceTest, floor) { + ASSERT_TRACE2("floor \"foo\"", + TypeError, + hintfmt("value is %s while a float was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.floor")); + + } + + + TEST_F(ErrorTraceTest, tryEval) { + } + + + TEST_F(ErrorTraceTest, getEnv) { + ASSERT_TRACE2("getEnv [ ]", + TypeError, + hintfmt("value is %s while a string was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.getEnv")); + + } + + + TEST_F(ErrorTraceTest, seq) { + } + + + TEST_F(ErrorTraceTest, deepSeq) { + } + + + TEST_F(ErrorTraceTest, trace) { + } + + + TEST_F(ErrorTraceTest, placeholder) { + ASSERT_TRACE2("placeholder []", + TypeError, + hintfmt("value is %s while a string was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.placeholder")); + + } + + + TEST_F(ErrorTraceTest, toPath) { + ASSERT_TRACE2("toPath []", + TypeError, + hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("while evaluating the first argument passed to builtins.toPath")); + + ASSERT_TRACE2("toPath \"foo\"", + EvalError, + hintfmt("string '%s' doesn't represent an absolute path", "foo"), + hintfmt("while evaluating the first argument passed to builtins.toPath")); + + } + + + TEST_F(ErrorTraceTest, storePath) { + ASSERT_TRACE2("storePath true", + TypeError, + hintfmt("cannot coerce %s to a string", "a Boolean"), + hintfmt("while evaluating the first argument passed to builtins.storePath")); + + } + + + TEST_F(ErrorTraceTest, pathExists) { + ASSERT_TRACE2("pathExists []", + TypeError, + hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("while realising the context of a path")); + + ASSERT_TRACE2("pathExists \"zorglub\"", + EvalError, + hintfmt("string '%s' doesn't represent an absolute path", "zorglub"), + hintfmt("while realising the context of a path")); + + } + + + TEST_F(ErrorTraceTest, baseNameOf) { + ASSERT_TRACE2("baseNameOf []", + TypeError, + hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("while evaluating the first argument passed to builtins.baseNameOf")); + + } + + + TEST_F(ErrorTraceTest, dirOf) { + } + + + TEST_F(ErrorTraceTest, readFile) { + } + + + TEST_F(ErrorTraceTest, findFile) { + } + + + TEST_F(ErrorTraceTest, hashFile) { + } + + + TEST_F(ErrorTraceTest, readDir) { + } + + + TEST_F(ErrorTraceTest, toXML) { + } + + + TEST_F(ErrorTraceTest, toJSON) { + } + + + TEST_F(ErrorTraceTest, fromJSON) { + } + + + TEST_F(ErrorTraceTest, toFile) { + } + + + TEST_F(ErrorTraceTest, filterSource) { + ASSERT_TRACE2("filterSource [] []", + TypeError, + hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + + ASSERT_TRACE2("filterSource [] \"foo\"", + EvalError, + hintfmt("string '%s' doesn't represent an absolute path", "foo"), + hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + + ASSERT_TRACE2("filterSource [] ./.", + TypeError, + hintfmt("value is %s while a function was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.filterSource")); + + // Usupported by store "dummy" + + // ASSERT_TRACE2("filterSource (_: 1) ./.", + // TypeError, + // hintfmt("attempt to call something which is not a function but %s", "an integer"), + // hintfmt("while adding path '/home/layus/projects/nix'")); + + // ASSERT_TRACE2("filterSource (_: _: 1) ./.", + // TypeError, + // hintfmt("value is %s while a Boolean was expected", "an integer"), + // hintfmt("while evaluating the return value of the path filter function")); + + } + + + TEST_F(ErrorTraceTest, path) { + } + + + TEST_F(ErrorTraceTest, attrNames) { + ASSERT_TRACE2("attrNames []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the argument passed to builtins.attrNames")); + + } + + + TEST_F(ErrorTraceTest, attrValues) { + ASSERT_TRACE2("attrValues []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the argument passed to builtins.attrValues")); + + } + + + TEST_F(ErrorTraceTest, getAttr) { + ASSERT_TRACE2("getAttr [] []", + TypeError, + hintfmt("value is %s while a string was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.getAttr")); + + ASSERT_TRACE2("getAttr \"foo\" []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the second argument passed to builtins.getAttr")); + + ASSERT_TRACE2("getAttr \"foo\" {}", + TypeError, + hintfmt("attribute '%s' missing", "foo"), + hintfmt("in the attribute set under consideration")); + + } + + + TEST_F(ErrorTraceTest, unsafeGetAttrPos) { + } + + + TEST_F(ErrorTraceTest, hasAttr) { + ASSERT_TRACE2("hasAttr [] []", + TypeError, + hintfmt("value is %s while a string was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.hasAttr")); + + ASSERT_TRACE2("hasAttr \"foo\" []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the second argument passed to builtins.hasAttr")); + + } + + + TEST_F(ErrorTraceTest, isAttrs) { + } + + + TEST_F(ErrorTraceTest, removeAttrs) { + ASSERT_TRACE2("removeAttrs \"\" \"\"", + TypeError, + hintfmt("value is %s while a set was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.removeAttrs")); + + ASSERT_TRACE2("removeAttrs \"\" [ 1 ]", + TypeError, + hintfmt("value is %s while a set was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.removeAttrs")); + + ASSERT_TRACE2("removeAttrs \"\" [ \"1\" ]", + TypeError, + hintfmt("value is %s while a set was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.removeAttrs")); + + } + + + TEST_F(ErrorTraceTest, listToAttrs) { + ASSERT_TRACE2("listToAttrs 1", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the argument passed to builtins.listToAttrs")); + + ASSERT_TRACE2("listToAttrs [ 1 ]", + TypeError, + hintfmt("value is %s while a set was expected", "an integer"), + hintfmt("while evaluating an element of the list passed to builtins.listToAttrs")); + + ASSERT_TRACE2("listToAttrs [ {} ]", + TypeError, + hintfmt("attribute '%s' missing", "name"), + hintfmt("in a {name=...; value=...;} pair")); + + ASSERT_TRACE2("listToAttrs [ { name = 1; } ]", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs")); + + ASSERT_TRACE2("listToAttrs [ { name = \"foo\"; } ]", + TypeError, + hintfmt("attribute '%s' missing", "value"), + hintfmt("in a {name=...; value=...;} pair")); + + } + + + TEST_F(ErrorTraceTest, intersectAttrs) { + ASSERT_TRACE2("intersectAttrs [] []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.intersectAttrs")); + + ASSERT_TRACE2("intersectAttrs {} []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the second argument passed to builtins.intersectAttrs")); + + } + + + TEST_F(ErrorTraceTest, catAttrs) { + ASSERT_TRACE2("catAttrs [] {}", + TypeError, + hintfmt("value is %s while a string was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.catAttrs")); + + ASSERT_TRACE2("catAttrs \"foo\" {}", + TypeError, + hintfmt("value is %s while a list was expected", "a set"), + hintfmt("while evaluating the second argument passed to builtins.catAttrs")); + + ASSERT_TRACE2("catAttrs \"foo\" [ 1 ]", + TypeError, + hintfmt("value is %s while a set was expected", "an integer"), + hintfmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); + + ASSERT_TRACE2("catAttrs \"foo\" [ { foo = 1; } 1 { bar = 5;} ]", + TypeError, + hintfmt("value is %s while a set was expected", "an integer"), + hintfmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); + + } + + + TEST_F(ErrorTraceTest, functionArgs) { + ASSERT_TRACE1("functionArgs {}", + TypeError, + hintfmt("'functionArgs' requires a function")); + + } + + + TEST_F(ErrorTraceTest, mapAttrs) { + ASSERT_TRACE2("mapAttrs [] []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the second argument passed to builtins.mapAttrs")); + + // XXX: defered + // ASSERT_TRACE2("mapAttrs \"\" { foo.bar = 1; }", + // TypeError, + // hintfmt("attempt to call something which is not a function but %s", "a string"), + // hintfmt("while evaluating the attribute 'foo'")); + + // ASSERT_TRACE2("mapAttrs (x: x + \"1\") { foo.bar = 1; }", + // TypeError, + // hintfmt("attempt to call something which is not a function but %s", "a string"), + // hintfmt("while evaluating the attribute 'foo'")); + + // ASSERT_TRACE2("mapAttrs (x: y: x + 1) { foo.bar = 1; }", + // TypeError, + // hintfmt("cannot coerce %s to a string", "an integer"), + // hintfmt("while evaluating a path segment")); + + } + + + TEST_F(ErrorTraceTest, zipAttrsWith) { + ASSERT_TRACE2("zipAttrsWith [] [ 1 ]", + TypeError, + hintfmt("value is %s while a function was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.zipAttrsWith")); + + ASSERT_TRACE2("zipAttrsWith (_: 1) [ 1 ]", + TypeError, + hintfmt("value is %s while a set was expected", "an integer"), + hintfmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith")); + + // XXX: How to properly tell that the fucntion takes two arguments ? + // The same question also applies to sort, and maybe others. + // Due to lazyness, we only create a thunk, and it fails later on. + // ASSERT_TRACE2("zipAttrsWith (_: 1) [ { foo = 1; } ]", + // TypeError, + // hintfmt("attempt to call something which is not a function but %s", "an integer"), + // hintfmt("while evaluating the attribute 'foo'")); + + // XXX: Also deferred deeply + // ASSERT_TRACE2("zipAttrsWith (a: b: a + b) [ { foo = 1; } { foo = 2; } ]", + // TypeError, + // hintfmt("cannot coerce %s to a string", "a list"), + // hintfmt("while evaluating a path segment")); + + } + + + TEST_F(ErrorTraceTest, isList) { + } + + + TEST_F(ErrorTraceTest, elemAt) { + ASSERT_TRACE2("elemAt \"foo\" (-1)", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.elemAt")); + + ASSERT_TRACE1("elemAt [] (-1)", + Error, + hintfmt("list index %d is out of bounds", -1)); + + ASSERT_TRACE1("elemAt [\"foo\"] 3", + Error, + hintfmt("list index %d is out of bounds", 3)); + + } + + + TEST_F(ErrorTraceTest, head) { + ASSERT_TRACE2("head 1", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.elemAt")); + + ASSERT_TRACE1("head []", + Error, + hintfmt("list index %d is out of bounds", 0)); + + } + + + TEST_F(ErrorTraceTest, tail) { + ASSERT_TRACE2("tail 1", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.tail")); + + ASSERT_TRACE1("tail []", + Error, + hintfmt("'tail' called on an empty list")); + + } + + + TEST_F(ErrorTraceTest, map) { + ASSERT_TRACE2("map 1 \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.map")); + + ASSERT_TRACE2("map 1 [ 1 ]", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.map")); + + } + + + TEST_F(ErrorTraceTest, filter) { + ASSERT_TRACE2("filter 1 \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.filter")); + + ASSERT_TRACE2("filter 1 [ \"foo\" ]", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.filter")); + + ASSERT_TRACE2("filter (_: 5) [ \"foo\" ]", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the return value of the filtering function passed to builtins.filter")); + + } + + + TEST_F(ErrorTraceTest, elem) { + ASSERT_TRACE2("elem 1 \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.elem")); + + } + + + TEST_F(ErrorTraceTest, concatLists) { + ASSERT_TRACE2("concatLists 1", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.concatLists")); + + ASSERT_TRACE2("concatLists [ 1 ]", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating a value of the list passed to builtins.concatLists")); + + ASSERT_TRACE2("concatLists [ [1] \"foo\" ]", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating a value of the list passed to builtins.concatLists")); + + } + + + TEST_F(ErrorTraceTest, length) { + ASSERT_TRACE2("length 1", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.length")); + + ASSERT_TRACE2("length \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.length")); + + } + + + TEST_F(ErrorTraceTest, foldlPrime) { + ASSERT_TRACE2("foldl' 1 \"foo\" true", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.foldlStrict")); + + ASSERT_TRACE2("foldl' (_: 1) \"foo\" true", + TypeError, + hintfmt("value is %s while a list was expected", "a Boolean"), + hintfmt("while evaluating the third argument passed to builtins.foldlStrict")); + + ASSERT_TRACE1("foldl' (_: 1) \"foo\" [ true ]", + TypeError, + hintfmt("attempt to call something which is not a function but %s", "an integer")); + + ASSERT_TRACE2("foldl' (a: b: a && b) \"foo\" [ true ]", + TypeError, + hintfmt("value is %s while a Boolean was expected", "a string"), + hintfmt("in the left operand of the AND (&&) operator")); + + } + + + TEST_F(ErrorTraceTest, any) { + ASSERT_TRACE2("any 1 \"foo\"", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.any")); + + ASSERT_TRACE2("any (_: 1) \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.any")); + + ASSERT_TRACE2("any (_: 1) [ \"foo\" ]", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the return value of the function passed to builtins.any")); + + } + + + TEST_F(ErrorTraceTest, all) { + ASSERT_TRACE2("all 1 \"foo\"", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.all")); + + ASSERT_TRACE2("all (_: 1) \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.all")); + + ASSERT_TRACE2("all (_: 1) [ \"foo\" ]", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the return value of the function passed to builtins.all")); + + } + + + TEST_F(ErrorTraceTest, genList) { + ASSERT_TRACE2("genList 1 \"foo\"", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.genList")); + + ASSERT_TRACE2("genList 1 2", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.genList", "an integer")); + + // XXX: defered + // ASSERT_TRACE2("genList (x: x + \"foo\") 2 #TODO", + // TypeError, + // hintfmt("cannot add %s to an integer", "a string"), + // hintfmt("while evaluating anonymous lambda")); + + ASSERT_TRACE1("genList false (-3)", + EvalError, + hintfmt("cannot create list of size %d", -3)); + + } + + + TEST_F(ErrorTraceTest, sort) { + ASSERT_TRACE2("sort 1 \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.sort")); + + ASSERT_TRACE2("sort 1 [ \"foo\" ]", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.sort")); + + ASSERT_TRACE1("sort (_: 1) [ \"foo\" \"bar\" ]", + TypeError, + hintfmt("attempt to call something which is not a function but %s", "an integer")); + + ASSERT_TRACE2("sort (_: _: 1) [ \"foo\" \"bar\" ]", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the return value of the sorting function passed to builtins.sort")); + + // XXX: Trace too deep, need better asserts + // ASSERT_TRACE1("sort (a: b: a <= b) [ \"foo\" {} ] # TODO", + // TypeError, + // hintfmt("cannot compare %s with %s", "a string", "a set")); + + // ASSERT_TRACE1("sort (a: b: a <= b) [ {} {} ] # TODO", + // TypeError, + // hintfmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); + + } + + + TEST_F(ErrorTraceTest, partition) { + ASSERT_TRACE2("partition 1 \"foo\"", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.partition")); + + ASSERT_TRACE2("partition (_: 1) \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.partition")); + + ASSERT_TRACE2("partition (_: 1) [ \"foo\" ]", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the return value of the partition function passed to builtins.partition")); + + } + + + TEST_F(ErrorTraceTest, groupBy) { + ASSERT_TRACE2("groupBy 1 \"foo\"", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.groupBy")); + + ASSERT_TRACE2("groupBy (_: 1) \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.groupBy")); + + ASSERT_TRACE2("groupBy (x: x) [ \"foo\" \"bar\" 1 ]", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the return value of the grouping function passed to builtins.groupBy")); + + } + + + TEST_F(ErrorTraceTest, concatMap) { + ASSERT_TRACE2("concatMap 1 \"foo\"", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.concatMap")); + + ASSERT_TRACE2("concatMap (x: 1) \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.concatMap")); + + ASSERT_TRACE2("concatMap (x: 1) [ \"foo\" ] # TODO", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + + ASSERT_TRACE2("concatMap (x: \"foo\") [ 1 2 ] # TODO", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + + } + + + TEST_F(ErrorTraceTest, add) { + ASSERT_TRACE2("add \"foo\" 1", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the first argument of the addition")); + + ASSERT_TRACE2("add 1 \"foo\"", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the second argument of the addition")); + + } + + + TEST_F(ErrorTraceTest, sub) { + ASSERT_TRACE2("sub \"foo\" 1", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the first argument of the subtraction")); + + ASSERT_TRACE2("sub 1 \"foo\"", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the second argument of the subtraction")); + + } + + + TEST_F(ErrorTraceTest, mul) { + ASSERT_TRACE2("mul \"foo\" 1", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the first argument of the multiplication")); + + ASSERT_TRACE2("mul 1 \"foo\"", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the second argument of the multiplication")); + + } + + + TEST_F(ErrorTraceTest, div) { + ASSERT_TRACE2("div \"foo\" 1 # TODO: an integer was expected -> a number", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the first operand of the division")); + + ASSERT_TRACE2("div 1 \"foo\"", + TypeError, + hintfmt("value is %s while a float was expected", "a string"), + hintfmt("while evaluating the second operand of the division")); + + ASSERT_TRACE1("div \"foo\" 0", + EvalError, + hintfmt("division by zero")); + + } + + + TEST_F(ErrorTraceTest, bitAnd) { + ASSERT_TRACE2("bitAnd 1.1 2", + TypeError, + hintfmt("value is %s while an integer was expected", "a float"), + hintfmt("while evaluating the first argument passed to builtins.bitAnd")); + + ASSERT_TRACE2("bitAnd 1 2.2", + TypeError, + hintfmt("value is %s while an integer was expected", "a float"), + hintfmt("while evaluating the second argument passed to builtins.bitAnd")); + + } + + + TEST_F(ErrorTraceTest, bitOr) { + ASSERT_TRACE2("bitOr 1.1 2", + TypeError, + hintfmt("value is %s while an integer was expected", "a float"), + hintfmt("while evaluating the first argument passed to builtins.bitOr")); + + ASSERT_TRACE2("bitOr 1 2.2", + TypeError, + hintfmt("value is %s while an integer was expected", "a float"), + hintfmt("while evaluating the second argument passed to builtins.bitOr")); + + } + + + TEST_F(ErrorTraceTest, bitXor) { + ASSERT_TRACE2("bitXor 1.1 2", + TypeError, + hintfmt("value is %s while an integer was expected", "a float"), + hintfmt("while evaluating the first argument passed to builtins.bitXor")); + + ASSERT_TRACE2("bitXor 1 2.2", + TypeError, + hintfmt("value is %s while an integer was expected", "a float"), + hintfmt("while evaluating the second argument passed to builtins.bitXor")); + + } + + + TEST_F(ErrorTraceTest, lessThan) { + ASSERT_TRACE1("lessThan 1 \"foo\"", + EvalError, + hintfmt("cannot compare %s with %s", "an integer", "a string")); + + ASSERT_TRACE1("lessThan {} {}", + EvalError, + hintfmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); + + ASSERT_TRACE2("lessThan [ 1 2 ] [ \"foo\" ]", + EvalError, + hintfmt("cannot compare %s with %s", "an integer", "a string"), + hintfmt("while comparing two list elements")); + + } + + + TEST_F(ErrorTraceTest, toString) { + ASSERT_TRACE2("toString { a = 1; }", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating the first argument passed to builtins.toString")); + + } + + + TEST_F(ErrorTraceTest, substring) { + ASSERT_TRACE2("substring {} \"foo\" true", + TypeError, + hintfmt("value is %s while an integer was expected", "a set"), + hintfmt("while evaluating the first argument (the start offset) passed to builtins.substring")); + + ASSERT_TRACE2("substring 3 \"foo\" true", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the second argument (the substring length) passed to builtins.substring")); + + ASSERT_TRACE2("substring 0 3 {}", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating the third argument (the string) passed to builtins.substring")); + + ASSERT_TRACE1("substring (-3) 3 \"sometext\"", + EvalError, + hintfmt("negative start position in 'substring'")); + + } + + + TEST_F(ErrorTraceTest, stringLength) { + ASSERT_TRACE2("stringLength {} # TODO: context is missing ???", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating the argument passed to builtins.stringLength")); + + } + + + TEST_F(ErrorTraceTest, hashString) { + ASSERT_TRACE2("hashString 1 {}", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.hashString")); + + ASSERT_TRACE1("hashString \"foo\" \"content\"", + UsageError, + hintfmt("unknown hash algorithm '%s'", "foo")); + + ASSERT_TRACE2("hashString \"sha256\" {}", + TypeError, + hintfmt("value is %s while a string was expected", "a set"), + hintfmt("while evaluating the second argument passed to builtins.hashString")); + + } + + + TEST_F(ErrorTraceTest, match) { + ASSERT_TRACE2("match 1 {}", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.match")); + + ASSERT_TRACE2("match \"foo\" {}", + TypeError, + hintfmt("value is %s while a string was expected", "a set"), + hintfmt("while evaluating the second argument passed to builtins.match")); + + ASSERT_TRACE1("match \"(.*\" \"\"", + EvalError, + hintfmt("invalid regular expression '%s'", "(.*")); + + } + + + TEST_F(ErrorTraceTest, split) { + ASSERT_TRACE2("split 1 {}", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.split")); + + ASSERT_TRACE2("split \"foo\" {}", + TypeError, + hintfmt("value is %s while a string was expected", "a set"), + hintfmt("while evaluating the second argument passed to builtins.split")); + + ASSERT_TRACE1("split \"f(o*o\" \"1foo2\"", + EvalError, + hintfmt("invalid regular expression '%s'", "f(o*o")); + + } + + + TEST_F(ErrorTraceTest, concatStringsSep) { + ASSERT_TRACE2("concatStringsSep 1 {}", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument (the separator string) passed to builtins.concatStringsSep")); + + ASSERT_TRACE2("concatStringsSep \"foo\" {}", + TypeError, + hintfmt("value is %s while a list was expected", "a set"), + hintfmt("while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep")); + + ASSERT_TRACE2("concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", + TypeError, + hintfmt("cannot coerce %s to a string", "an integer"), + hintfmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); + + } + + + TEST_F(ErrorTraceTest, parseDrvName) { + ASSERT_TRACE2("parseDrvName 1", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.parseDrvName")); + + } + + + TEST_F(ErrorTraceTest, compareVersions) { + ASSERT_TRACE2("compareVersions 1 {}", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.compareVersions")); + + ASSERT_TRACE2("compareVersions \"abd\" {}", + TypeError, + hintfmt("value is %s while a string was expected", "a set"), + hintfmt("while evaluating the second argument passed to builtins.compareVersions")); + + } + + + TEST_F(ErrorTraceTest, splitVersion) { + ASSERT_TRACE2("splitVersion 1", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.splitVersion")); + + } + + + TEST_F(ErrorTraceTest, traceVerbose) { + } + + + /* // Needs different ASSERTs + TEST_F(ErrorTraceTest, derivationStrict) { + ASSERT_TRACE2("derivationStrict \"\"", + TypeError, + hintfmt("value is %s while a set was expected", "a string"), + hintfmt("while evaluating the argument passed to builtins.derivationStrict")); + + ASSERT_TRACE2("derivationStrict {}", + TypeError, + hintfmt("attribute '%s' missing", "name"), + hintfmt("in the attrset passed as argument to builtins.derivationStrict")); + + ASSERT_TRACE2("derivationStrict { name = 1; }", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the `name` attribute passed to builtins.derivationStrict")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; }", + TypeError, + hintfmt("required attribute 'builder' missing"), + hintfmt("while evaluating derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; __structuredAttrs = 15; }", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; __ignoreNulls = 15; }", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; outputHashMode = 15; }", + TypeError, + hintfmt("invalid value '15' for 'outputHashMode' attribute"), + hintfmt("while evaluating the attribute 'outputHashMode' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; outputHashMode = \"custom\"; }", + TypeError, + hintfmt("invalid value 'custom' for 'outputHashMode' attribute"), + hintfmt("while evaluating the attribute 'outputHashMode' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = {}; }", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating the attribute 'system' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drv\"; }", + TypeError, + hintfmt("invalid derivation output name 'drv'"), + hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = []; }", + TypeError, + hintfmt("derivation cannot have an empty set of outputs"), + hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"drv\" ]; }", + TypeError, + hintfmt("invalid derivation output name 'drv'"), + hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"out\" \"out\" ]; }", + TypeError, + hintfmt("duplicate derivation output 'out'"), + hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __contentAddressed = \"true\"; }", + TypeError, + hintfmt("value is %s while a Boolean was expected", "a string"), + hintfmt("while evaluating the attribute '__contentAddressed' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", + TypeError, + hintfmt("value is %s while a Boolean was expected", "a string"), + hintfmt("while evaluating the attribute '__impure' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", + TypeError, + hintfmt("value is %s while a Boolean was expected", "a string"), + hintfmt("while evaluating the attribute '__impure' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = \"foo\"; }", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the attribute 'args' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating an element of the argument list")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating an element of the argument list")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating the attribute 'FOO' of derivation 'foo'")); + + } + */ + +} /* namespace nix */ diff -Nru nix-2.8.0/src/libexpr/tests/json.cc nix-2.16.1+dfsg/src/libexpr/tests/json.cc --- nix-2.8.0/src/libexpr/tests/json.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/tests/json.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,68 @@ +#include "tests/libexpr.hh" +#include "value-to-json.hh" + +namespace nix { +// Testing the conversion to JSON + + class JSONValueTest : public LibExprTest { + protected: + std::string getJSONValue(Value& value) { + std::stringstream ss; + NixStringContext ps; + printValueAsJSON(state, true, value, noPos, ss, ps); + return ss.str(); + } + }; + + TEST_F(JSONValueTest, null) { + Value v; + v.mkNull(); + ASSERT_EQ(getJSONValue(v), "null"); + } + + TEST_F(JSONValueTest, BoolFalse) { + Value v; + v.mkBool(false); + ASSERT_EQ(getJSONValue(v),"false"); + } + + TEST_F(JSONValueTest, BoolTrue) { + Value v; + v.mkBool(true); + ASSERT_EQ(getJSONValue(v), "true"); + } + + TEST_F(JSONValueTest, IntPositive) { + Value v; + v.mkInt(100); + ASSERT_EQ(getJSONValue(v), "100"); + } + + TEST_F(JSONValueTest, IntNegative) { + Value v; + v.mkInt(-100); + ASSERT_EQ(getJSONValue(v), "-100"); + } + + TEST_F(JSONValueTest, String) { + Value v; + v.mkString("test"); + ASSERT_EQ(getJSONValue(v), "\"test\""); + } + + TEST_F(JSONValueTest, StringQuotes) { + Value v; + + v.mkString("test\""); + ASSERT_EQ(getJSONValue(v), "\"test\\\"\""); + } + + // The dummy store doesn't support writing files. Fails with this exception message: + // C++ exception with description "error: operation 'addToStoreFromDump' is + // not supported by store 'dummy'" thrown in the test body. + TEST_F(JSONValueTest, DISABLED_Path) { + Value v; + v.mkPath("test"); + ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\""); + } +} /* namespace nix */ diff -Nru nix-2.8.0/src/libexpr/tests/libexpr.hh nix-2.16.1+dfsg/src/libexpr/tests/libexpr.hh --- nix-2.8.0/src/libexpr/tests/libexpr.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/tests/libexpr.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,140 @@ +#pragma once +///@file + +#include +#include + +#include "value.hh" +#include "nixexpr.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "store-api.hh" + +#include "tests/libstore.hh" + +namespace nix { + class LibExprTest : public LibStoreTest { + public: + static void SetUpTestSuite() { + LibStoreTest::SetUpTestSuite(); + initGC(); + } + + protected: + LibExprTest() + : LibStoreTest() + , state({}, store) + { + } + Value eval(std::string input, bool forceValue = true) { + Value v; + Expr * e = state.parseExprFromString(input, state.rootPath(CanonPath::root)); + assert(e); + state.eval(e, v); + if (forceValue) + state.forceValue(v, noPos); + return v; + } + + Symbol createSymbol(const char * value) { + return state.symbols.create(value); + } + + EvalState state; + }; + + MATCHER(IsListType, "") { + return arg != nList; + } + + MATCHER(IsList, "") { + return arg.type() == nList; + } + + MATCHER(IsString, "") { + return arg.type() == nString; + } + + MATCHER(IsNull, "") { + return arg.type() == nNull; + } + + MATCHER(IsThunk, "") { + return arg.type() == nThunk; + } + + MATCHER(IsAttrs, "") { + return arg.type() == nAttrs; + } + + MATCHER_P(IsStringEq, s, fmt("The string is equal to \"%1%\"", s)) { + if (arg.type() != nString) { + return false; + } + return std::string_view(arg.string.s) == s; + } + + MATCHER_P(IsIntEq, v, fmt("The string is equal to \"%1%\"", v)) { + if (arg.type() != nInt) { + return false; + } + return arg.integer == v; + } + + MATCHER_P(IsFloatEq, v, fmt("The float is equal to \"%1%\"", v)) { + if (arg.type() != nFloat) { + return false; + } + return arg.fpoint == v; + } + + MATCHER(IsTrue, "") { + if (arg.type() != nBool) { + return false; + } + return arg.boolean == true; + } + + MATCHER(IsFalse, "") { + if (arg.type() != nBool) { + return false; + } + return arg.boolean == false; + } + + MATCHER_P(IsPathEq, p, fmt("Is a path equal to \"%1%\"", p)) { + if (arg.type() != nPath) { + *result_listener << "Expected a path got " << arg.type(); + return false; + } else if (std::string_view(arg.string.s) != p) { + *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.string.s; + return false; + } + return true; + } + + + MATCHER_P(IsListOfSize, n, fmt("Is a list of size [%1%]", n)) { + if (arg.type() != nList) { + *result_listener << "Expected list got " << arg.type(); + return false; + } else if (arg.listSize() != (size_t)n) { + *result_listener << "Expected as list of size " << n << " got " << arg.listSize(); + return false; + } + return true; + } + + MATCHER_P(IsAttrsOfSize, n, fmt("Is a set of size [%1%]", n)) { + if (arg.type() != nAttrs) { + *result_listener << "Expected set got " << arg.type(); + return false; + } else if (arg.attrs->size() != (size_t)n) { + *result_listener << "Expected a set with " << n << " attributes but got " << arg.attrs->size(); + return false; + } + return true; + } + + +} /* namespace nix */ diff -Nru nix-2.8.0/src/libexpr/tests/local.mk nix-2.16.1+dfsg/src/libexpr/tests/local.mk --- nix-2.8.0/src/libexpr/tests/local.mk 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/tests/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,19 @@ +check: libexpr-tests_RUN + +programs += libexpr-tests + +libexpr-tests_NAME := libnixexpr-tests + +libexpr-tests_DIR := $(d) + +libexpr-tests_INSTALL_DIR := + +libexpr-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) + +libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers + +libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers + +libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock diff -Nru nix-2.8.0/src/libexpr/tests/primops.cc nix-2.16.1+dfsg/src/libexpr/tests/primops.cc --- nix-2.8.0/src/libexpr/tests/primops.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/tests/primops.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,832 @@ +#include +#include + +#include "tests/libexpr.hh" + +namespace nix { + class CaptureLogger : public Logger + { + std::ostringstream oss; + + public: + CaptureLogger() {} + + std::string get() const { + return oss.str(); + } + + void log(Verbosity lvl, std::string_view s) override { + oss << s << std::endl; + } + + void logEI(const ErrorInfo & ei) override { + showErrorInfo(oss, ei, loggerSettings.showTrace.get()); + } + }; + + class CaptureLogging { + Logger * oldLogger; + std::unique_ptr tempLogger; + public: + CaptureLogging() : tempLogger(std::make_unique()) { + oldLogger = logger; + logger = tempLogger.get(); + } + + ~CaptureLogging() { + logger = oldLogger; + } + + std::string get() const { + return tempLogger->get(); + } + }; + + + // Testing eval of PrimOp's + class PrimOpTest : public LibExprTest {}; + + + TEST_F(PrimOpTest, throw) { + ASSERT_THROW(eval("throw \"foo\""), ThrownError); + } + + TEST_F(PrimOpTest, abort) { + ASSERT_THROW(eval("abort \"abort\""), Abort); + } + + TEST_F(PrimOpTest, ceil) { + auto v = eval("builtins.ceil 1.9"); + ASSERT_THAT(v, IsIntEq(2)); + } + + TEST_F(PrimOpTest, floor) { + auto v = eval("builtins.floor 1.9"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(PrimOpTest, tryEvalFailure) { + auto v = eval("builtins.tryEval (throw \"\")"); + ASSERT_THAT(v, IsAttrsOfSize(2)); + auto s = createSymbol("success"); + auto p = v.attrs->get(s); + ASSERT_NE(p, nullptr); + ASSERT_THAT(*p->value, IsFalse()); + } + + TEST_F(PrimOpTest, tryEvalSuccess) { + auto v = eval("builtins.tryEval 123"); + ASSERT_THAT(v, IsAttrs()); + auto s = createSymbol("success"); + auto p = v.attrs->get(s); + ASSERT_NE(p, nullptr); + ASSERT_THAT(*p->value, IsTrue()); + s = createSymbol("value"); + p = v.attrs->get(s); + ASSERT_NE(p, nullptr); + ASSERT_THAT(*p->value, IsIntEq(123)); + } + + TEST_F(PrimOpTest, getEnv) { + setenv("_NIX_UNIT_TEST_ENV_VALUE", "test value", 1); + auto v = eval("builtins.getEnv \"_NIX_UNIT_TEST_ENV_VALUE\""); + ASSERT_THAT(v, IsStringEq("test value")); + } + + TEST_F(PrimOpTest, seq) { + ASSERT_THROW(eval("let x = throw \"test\"; in builtins.seq x { }"), ThrownError); + } + + TEST_F(PrimOpTest, seqNotDeep) { + auto v = eval("let x = { z = throw \"test\"; }; in builtins.seq x { }"); + ASSERT_THAT(v, IsAttrs()); + } + + TEST_F(PrimOpTest, deepSeq) { + ASSERT_THROW(eval("let x = { z = throw \"test\"; }; in builtins.deepSeq x { }"), ThrownError); + } + + TEST_F(PrimOpTest, trace) { + CaptureLogging l; + auto v = eval("builtins.trace \"test string 123\" 123"); + ASSERT_THAT(v, IsIntEq(123)); + auto text = l.get(); + ASSERT_NE(text.find("test string 123"), std::string::npos); + } + + TEST_F(PrimOpTest, placeholder) { + auto v = eval("builtins.placeholder \"out\""); + ASSERT_THAT(v, IsStringEq("/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9")); + } + + TEST_F(PrimOpTest, baseNameOf) { + auto v = eval("builtins.baseNameOf /some/path"); + ASSERT_THAT(v, IsStringEq("path")); + } + + TEST_F(PrimOpTest, dirOf) { + auto v = eval("builtins.dirOf /some/path"); + ASSERT_THAT(v, IsPathEq("/some")); + } + + TEST_F(PrimOpTest, attrValues) { + auto v = eval("builtins.attrValues { x = \"foo\"; a = 1; }"); + ASSERT_THAT(v, IsListOfSize(2)); + ASSERT_THAT(*v.listElems()[0], IsIntEq(1)); + ASSERT_THAT(*v.listElems()[1], IsStringEq("foo")); + } + + TEST_F(PrimOpTest, getAttr) { + auto v = eval("builtins.getAttr \"x\" { x = \"foo\"; }"); + ASSERT_THAT(v, IsStringEq("foo")); + } + + TEST_F(PrimOpTest, getAttrNotFound) { + // FIXME: TypeError is really bad here, also the error wording is worse + // than on Nix <=2.3 + ASSERT_THROW(eval("builtins.getAttr \"y\" { }"), TypeError); + } + + TEST_F(PrimOpTest, unsafeGetAttrPos) { + // The `y` attribute is at position + const char* expr = "builtins.unsafeGetAttrPos \"y\" { y = \"x\"; }"; + auto v = eval(expr); + ASSERT_THAT(v, IsNull()); + } + + TEST_F(PrimOpTest, hasAttr) { + auto v = eval("builtins.hasAttr \"x\" { x = 1; }"); + ASSERT_THAT(v, IsTrue()); + } + + TEST_F(PrimOpTest, hasAttrNotFound) { + auto v = eval("builtins.hasAttr \"x\" { }"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(PrimOpTest, isAttrs) { + auto v = eval("builtins.isAttrs {}"); + ASSERT_THAT(v, IsTrue()); + } + + TEST_F(PrimOpTest, isAttrsFalse) { + auto v = eval("builtins.isAttrs null"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(PrimOpTest, removeAttrs) { + auto v = eval("builtins.removeAttrs { x = 1; } [\"x\"]"); + ASSERT_THAT(v, IsAttrsOfSize(0)); + } + + TEST_F(PrimOpTest, removeAttrsRetains) { + auto v = eval("builtins.removeAttrs { x = 1; y = 2; } [\"x\"]"); + ASSERT_THAT(v, IsAttrsOfSize(1)); + ASSERT_NE(v.attrs->find(createSymbol("y")), nullptr); + } + + TEST_F(PrimOpTest, listToAttrsEmptyList) { + auto v = eval("builtins.listToAttrs []"); + ASSERT_THAT(v, IsAttrsOfSize(0)); + ASSERT_EQ(v.type(), nAttrs); + ASSERT_EQ(v.attrs->size(), 0); + } + + TEST_F(PrimOpTest, listToAttrsNotFieldName) { + ASSERT_THROW(eval("builtins.listToAttrs [{}]"), Error); + } + + TEST_F(PrimOpTest, listToAttrs) { + auto v = eval("builtins.listToAttrs [ { name = \"key\"; value = 123; } ]"); + ASSERT_THAT(v, IsAttrsOfSize(1)); + auto key = v.attrs->find(createSymbol("key")); + ASSERT_NE(key, nullptr); + ASSERT_THAT(*key->value, IsIntEq(123)); + } + + TEST_F(PrimOpTest, intersectAttrs) { + auto v = eval("builtins.intersectAttrs { a = 1; b = 2; } { b = 3; c = 4; }"); + ASSERT_THAT(v, IsAttrsOfSize(1)); + auto b = v.attrs->find(createSymbol("b")); + ASSERT_NE(b, nullptr); + ASSERT_THAT(*b->value, IsIntEq(3)); + } + + TEST_F(PrimOpTest, catAttrs) { + auto v = eval("builtins.catAttrs \"a\" [{a = 1;} {b = 0;} {a = 2;}]"); + ASSERT_THAT(v, IsListOfSize(2)); + ASSERT_THAT(*v.listElems()[0], IsIntEq(1)); + ASSERT_THAT(*v.listElems()[1], IsIntEq(2)); + } + + TEST_F(PrimOpTest, functionArgs) { + auto v = eval("builtins.functionArgs ({ x, y ? 123}: 1)"); + ASSERT_THAT(v, IsAttrsOfSize(2)); + + auto x = v.attrs->find(createSymbol("x")); + ASSERT_NE(x, nullptr); + ASSERT_THAT(*x->value, IsFalse()); + + auto y = v.attrs->find(createSymbol("y")); + ASSERT_NE(y, nullptr); + ASSERT_THAT(*y->value, IsTrue()); + } + + TEST_F(PrimOpTest, mapAttrs) { + auto v = eval("builtins.mapAttrs (name: value: value * 10) { a = 1; b = 2; }"); + ASSERT_THAT(v, IsAttrsOfSize(2)); + + auto a = v.attrs->find(createSymbol("a")); + ASSERT_NE(a, nullptr); + ASSERT_THAT(*a->value, IsThunk()); + state.forceValue(*a->value, noPos); + ASSERT_THAT(*a->value, IsIntEq(10)); + + auto b = v.attrs->find(createSymbol("b")); + ASSERT_NE(b, nullptr); + ASSERT_THAT(*b->value, IsThunk()); + state.forceValue(*b->value, noPos); + ASSERT_THAT(*b->value, IsIntEq(20)); + } + + TEST_F(PrimOpTest, isList) { + auto v = eval("builtins.isList []"); + ASSERT_THAT(v, IsTrue()); + } + + TEST_F(PrimOpTest, isListFalse) { + auto v = eval("builtins.isList null"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(PrimOpTest, elemtAt) { + auto v = eval("builtins.elemAt [0 1 2 3] 3"); + ASSERT_THAT(v, IsIntEq(3)); + } + + TEST_F(PrimOpTest, elemtAtOutOfBounds) { + ASSERT_THROW(eval("builtins.elemAt [0 1 2 3] 5"), Error); + } + + TEST_F(PrimOpTest, head) { + auto v = eval("builtins.head [ 3 2 1 0 ]"); + ASSERT_THAT(v, IsIntEq(3)); + } + + TEST_F(PrimOpTest, headEmpty) { + ASSERT_THROW(eval("builtins.head [ ]"), Error); + } + + TEST_F(PrimOpTest, headWrongType) { + ASSERT_THROW(eval("builtins.head { }"), Error); + } + + TEST_F(PrimOpTest, tail) { + auto v = eval("builtins.tail [ 3 2 1 0 ]"); + ASSERT_THAT(v, IsListOfSize(3)); + for (const auto [n, elem] : enumerate(v.listItems())) + ASSERT_THAT(*elem, IsIntEq(2 - static_cast(n))); + } + + TEST_F(PrimOpTest, tailEmpty) { + ASSERT_THROW(eval("builtins.tail []"), Error); + } + + TEST_F(PrimOpTest, map) { + auto v = eval("map (x: \"foo\" + x) [ \"bar\" \"bla\" \"abc\" ]"); + ASSERT_THAT(v, IsListOfSize(3)); + auto elem = v.listElems()[0]; + ASSERT_THAT(*elem, IsThunk()); + state.forceValue(*elem, noPos); + ASSERT_THAT(*elem, IsStringEq("foobar")); + + elem = v.listElems()[1]; + ASSERT_THAT(*elem, IsThunk()); + state.forceValue(*elem, noPos); + ASSERT_THAT(*elem, IsStringEq("foobla")); + + elem = v.listElems()[2]; + ASSERT_THAT(*elem, IsThunk()); + state.forceValue(*elem, noPos); + ASSERT_THAT(*elem, IsStringEq("fooabc")); + } + + TEST_F(PrimOpTest, filter) { + auto v = eval("builtins.filter (x: x == 2) [ 3 2 3 2 3 2 ]"); + ASSERT_THAT(v, IsListOfSize(3)); + for (const auto elem : v.listItems()) + ASSERT_THAT(*elem, IsIntEq(2)); + } + + TEST_F(PrimOpTest, elemTrue) { + auto v = eval("builtins.elem 3 [ 1 2 3 4 5 ]"); + ASSERT_THAT(v, IsTrue()); + } + + TEST_F(PrimOpTest, elemFalse) { + auto v = eval("builtins.elem 6 [ 1 2 3 4 5 ]"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(PrimOpTest, concatLists) { + auto v = eval("builtins.concatLists [[1 2] [3 4]]"); + ASSERT_THAT(v, IsListOfSize(4)); + for (const auto [i, elem] : enumerate(v.listItems())) + ASSERT_THAT(*elem, IsIntEq(static_cast(i)+1)); + } + + TEST_F(PrimOpTest, length) { + auto v = eval("builtins.length [ 1 2 3 ]"); + ASSERT_THAT(v, IsIntEq(3)); + } + + TEST_F(PrimOpTest, foldStrict) { + auto v = eval("builtins.foldl' (a: b: a + b) 0 [1 2 3]"); + ASSERT_THAT(v, IsIntEq(6)); + } + + TEST_F(PrimOpTest, anyTrue) { + auto v = eval("builtins.any (x: x == 2) [ 1 2 3 ]"); + ASSERT_THAT(v, IsTrue()); + } + + TEST_F(PrimOpTest, anyFalse) { + auto v = eval("builtins.any (x: x == 5) [ 1 2 3 ]"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(PrimOpTest, allTrue) { + auto v = eval("builtins.all (x: x > 0) [ 1 2 3 ]"); + ASSERT_THAT(v, IsTrue()); + } + + TEST_F(PrimOpTest, allFalse) { + auto v = eval("builtins.all (x: x <= 0) [ 1 2 3 ]"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(PrimOpTest, genList) { + auto v = eval("builtins.genList (x: x + 1) 3"); + ASSERT_EQ(v.type(), nList); + ASSERT_EQ(v.listSize(), 3); + for (const auto [i, elem] : enumerate(v.listItems())) { + ASSERT_THAT(*elem, IsThunk()); + state.forceValue(*elem, noPos); + ASSERT_THAT(*elem, IsIntEq(static_cast(i)+1)); + } + } + + TEST_F(PrimOpTest, sortLessThan) { + auto v = eval("builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]"); + ASSERT_EQ(v.type(), nList); + ASSERT_EQ(v.listSize(), 6); + + const std::vector numbers = { 42, 77, 147, 249, 483, 526 }; + for (const auto [n, elem] : enumerate(v.listItems())) + ASSERT_THAT(*elem, IsIntEq(numbers[n])); + } + + TEST_F(PrimOpTest, partition) { + auto v = eval("builtins.partition (x: x > 10) [1 23 9 3 42]"); + ASSERT_THAT(v, IsAttrsOfSize(2)); + + auto right = v.attrs->get(createSymbol("right")); + ASSERT_NE(right, nullptr); + ASSERT_THAT(*right->value, IsListOfSize(2)); + ASSERT_THAT(*right->value->listElems()[0], IsIntEq(23)); + ASSERT_THAT(*right->value->listElems()[1], IsIntEq(42)); + + auto wrong = v.attrs->get(createSymbol("wrong")); + ASSERT_NE(wrong, nullptr); + ASSERT_EQ(wrong->value->type(), nList); + ASSERT_EQ(wrong->value->listSize(), 3); + ASSERT_THAT(*wrong->value, IsListOfSize(3)); + ASSERT_THAT(*wrong->value->listElems()[0], IsIntEq(1)); + ASSERT_THAT(*wrong->value->listElems()[1], IsIntEq(9)); + ASSERT_THAT(*wrong->value->listElems()[2], IsIntEq(3)); + } + + TEST_F(PrimOpTest, concatMap) { + auto v = eval("builtins.concatMap (x: x ++ [0]) [ [1 2] [3 4] ]"); + ASSERT_EQ(v.type(), nList); + ASSERT_EQ(v.listSize(), 6); + + const std::vector numbers = { 1, 2, 0, 3, 4, 0 }; + for (const auto [n, elem] : enumerate(v.listItems())) + ASSERT_THAT(*elem, IsIntEq(numbers[n])); + } + + TEST_F(PrimOpTest, addInt) { + auto v = eval("builtins.add 3 5"); + ASSERT_THAT(v, IsIntEq(8)); + } + + TEST_F(PrimOpTest, addFloat) { + auto v = eval("builtins.add 3.0 5.0"); + ASSERT_THAT(v, IsFloatEq(8.0)); + } + + TEST_F(PrimOpTest, addFloatToInt) { + auto v = eval("builtins.add 3.0 5"); + ASSERT_THAT(v, IsFloatEq(8.0)); + + v = eval("builtins.add 3 5.0"); + ASSERT_THAT(v, IsFloatEq(8.0)); + } + + TEST_F(PrimOpTest, subInt) { + auto v = eval("builtins.sub 5 2"); + ASSERT_THAT(v, IsIntEq(3)); + } + + TEST_F(PrimOpTest, subFloat) { + auto v = eval("builtins.sub 5.0 2.0"); + ASSERT_THAT(v, IsFloatEq(3.0)); + } + + TEST_F(PrimOpTest, subFloatFromInt) { + auto v = eval("builtins.sub 5.0 2"); + ASSERT_THAT(v, IsFloatEq(3.0)); + + v = eval("builtins.sub 4 2.0"); + ASSERT_THAT(v, IsFloatEq(2.0)); + } + + TEST_F(PrimOpTest, mulInt) { + auto v = eval("builtins.mul 3 5"); + ASSERT_THAT(v, IsIntEq(15)); + } + + TEST_F(PrimOpTest, mulFloat) { + auto v = eval("builtins.mul 3.0 5.0"); + ASSERT_THAT(v, IsFloatEq(15.0)); + } + + TEST_F(PrimOpTest, mulFloatMixed) { + auto v = eval("builtins.mul 3 5.0"); + ASSERT_THAT(v, IsFloatEq(15.0)); + + v = eval("builtins.mul 2.0 5"); + ASSERT_THAT(v, IsFloatEq(10.0)); + } + + TEST_F(PrimOpTest, divInt) { + auto v = eval("builtins.div 5 (-1)"); + ASSERT_THAT(v, IsIntEq(-5)); + } + + TEST_F(PrimOpTest, divIntZero) { + ASSERT_THROW(eval("builtins.div 5 0"), EvalError); + } + + TEST_F(PrimOpTest, divFloat) { + auto v = eval("builtins.div 5.0 (-1)"); + ASSERT_THAT(v, IsFloatEq(-5.0)); + } + + TEST_F(PrimOpTest, divFloatZero) { + ASSERT_THROW(eval("builtins.div 5.0 0.0"), EvalError); + } + + TEST_F(PrimOpTest, bitOr) { + auto v = eval("builtins.bitOr 1 2"); + ASSERT_THAT(v, IsIntEq(3)); + } + + TEST_F(PrimOpTest, bitXor) { + auto v = eval("builtins.bitXor 3 2"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(PrimOpTest, lessThanFalse) { + auto v = eval("builtins.lessThan 3 1"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(PrimOpTest, lessThanTrue) { + auto v = eval("builtins.lessThan 1 3"); + ASSERT_THAT(v, IsTrue()); + } + + TEST_F(PrimOpTest, toStringAttrsThrows) { + ASSERT_THROW(eval("builtins.toString {}"), EvalError); + } + + TEST_F(PrimOpTest, toStringLambdaThrows) { + ASSERT_THROW(eval("builtins.toString (x: x)"), EvalError); + } + + class ToStringPrimOpTest : + public PrimOpTest, + public testing::WithParamInterface> + {}; + + TEST_P(ToStringPrimOpTest, toString) { + const auto [input, output] = GetParam(); + auto v = eval(input); + ASSERT_THAT(v, IsStringEq(output)); + } + +#define CASE(input, output) (std::make_tuple(std::string_view("builtins.toString " input), std::string_view(output))) + INSTANTIATE_TEST_SUITE_P( + toString, + ToStringPrimOpTest, + testing::Values( + CASE(R"("foo")", "foo"), + CASE(R"(1)", "1"), + CASE(R"([1 2 3])", "1 2 3"), + CASE(R"(.123)", "0.123000"), + CASE(R"(true)", "1"), + CASE(R"(false)", ""), + CASE(R"(null)", ""), + CASE(R"({ v = "bar"; __toString = self: self.v; })", "bar"), + CASE(R"({ v = "bar"; __toString = self: self.v; outPath = "foo"; })", "bar"), + CASE(R"({ outPath = "foo"; })", "foo"), + CASE(R"(./test)", "/test") + ) + ); +#undef CASE + + TEST_F(PrimOpTest, substring){ + auto v = eval("builtins.substring 0 3 \"nixos\""); + ASSERT_THAT(v, IsStringEq("nix")); + } + + TEST_F(PrimOpTest, substringSmallerString){ + auto v = eval("builtins.substring 0 3 \"n\""); + ASSERT_THAT(v, IsStringEq("n")); + } + + TEST_F(PrimOpTest, substringEmptyString){ + auto v = eval("builtins.substring 1 3 \"\""); + ASSERT_THAT(v, IsStringEq("")); + } + + TEST_F(PrimOpTest, stringLength) { + auto v = eval("builtins.stringLength \"123\""); + ASSERT_THAT(v, IsIntEq(3)); + } + TEST_F(PrimOpTest, hashStringMd5) { + auto v = eval("builtins.hashString \"md5\" \"asdf\""); + ASSERT_THAT(v, IsStringEq("912ec803b2ce49e4a541068d495ab570")); + } + + TEST_F(PrimOpTest, hashStringSha1) { + auto v = eval("builtins.hashString \"sha1\" \"asdf\""); + ASSERT_THAT(v, IsStringEq("3da541559918a808c2402bba5012f6c60b27661c")); + } + + TEST_F(PrimOpTest, hashStringSha256) { + auto v = eval("builtins.hashString \"sha256\" \"asdf\""); + ASSERT_THAT(v, IsStringEq("f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b")); + } + + TEST_F(PrimOpTest, hashStringSha512) { + auto v = eval("builtins.hashString \"sha512\" \"asdf\""); + ASSERT_THAT(v, IsStringEq("401b09eab3c013d4ca54922bb802bec8fd5318192b0a75f201d8b3727429080fb337591abd3e44453b954555b7a0812e1081c39b740293f765eae731f5a65ed1")); + } + + TEST_F(PrimOpTest, hashStringInvalidHashType) { + ASSERT_THROW(eval("builtins.hashString \"foobar\" \"asdf\""), Error); + } + + TEST_F(PrimOpTest, nixPath) { + auto v = eval("builtins.nixPath"); + ASSERT_EQ(v.type(), nList); + // We can't test much more as currently the EvalSettings are a global + // that we can't easily swap / replace + } + + TEST_F(PrimOpTest, langVersion) { + auto v = eval("builtins.langVersion"); + ASSERT_EQ(v.type(), nInt); + } + + TEST_F(PrimOpTest, storeDir) { + auto v = eval("builtins.storeDir"); + ASSERT_THAT(v, IsStringEq(settings.nixStore)); + } + + TEST_F(PrimOpTest, nixVersion) { + auto v = eval("builtins.nixVersion"); + ASSERT_THAT(v, IsStringEq(nixVersion)); + } + + TEST_F(PrimOpTest, currentSystem) { + auto v = eval("builtins.currentSystem"); + ASSERT_THAT(v, IsStringEq(settings.thisSystem.get())); + } + + TEST_F(PrimOpTest, derivation) { + auto v = eval("derivation"); + ASSERT_EQ(v.type(), nFunction); + ASSERT_TRUE(v.isLambda()); + ASSERT_NE(v.lambda.fun, nullptr); + ASSERT_TRUE(v.lambda.fun->hasFormals()); + } + + TEST_F(PrimOpTest, currentTime) { + auto v = eval("builtins.currentTime"); + ASSERT_EQ(v.type(), nInt); + ASSERT_TRUE(v.integer > 0); + } + + TEST_F(PrimOpTest, splitVersion) { + auto v = eval("builtins.splitVersion \"1.2.3git\""); + ASSERT_THAT(v, IsListOfSize(4)); + + const std::vector strings = { "1", "2", "3", "git" }; + for (const auto [n, p] : enumerate(v.listItems())) + ASSERT_THAT(*p, IsStringEq(strings[n])); + } + + class CompareVersionsPrimOpTest : + public PrimOpTest, + public testing::WithParamInterface> + {}; + + TEST_P(CompareVersionsPrimOpTest, compareVersions) { + auto [expression, expectation] = GetParam(); + auto v = eval(expression); + ASSERT_THAT(v, IsIntEq(expectation)); + } + +#define CASE(a, b, expected) (std::make_tuple("builtins.compareVersions \"" #a "\" \"" #b "\"", expected)) + INSTANTIATE_TEST_SUITE_P( + compareVersions, + CompareVersionsPrimOpTest, + testing::Values( + // The first two are weird cases. Intuition tells they should + // be the same but they aren't. + CASE(1.0, 1.0.0, -1), + CASE(1.0.0, 1.0, 1), + // the following are from the nix-env manual: + CASE(1.0, 2.3, -1), + CASE(2.1, 2.3, -1), + CASE(2.3, 2.3, 0), + CASE(2.5, 2.3, 1), + CASE(3.1, 2.3, 1), + CASE(2.3.1, 2.3, 1), + CASE(2.3.1, 2.3a, 1), + CASE(2.3pre1, 2.3, -1), + CASE(2.3pre3, 2.3pre12, -1), + CASE(2.3a, 2.3c, -1), + CASE(2.3pre1, 2.3c, -1), + CASE(2.3pre1, 2.3q, -1) + ) + ); +#undef CASE + + + class ParseDrvNamePrimOpTest : + public PrimOpTest, + public testing::WithParamInterface> + {}; + + TEST_P(ParseDrvNamePrimOpTest, parseDrvName) { + auto [input, expectedName, expectedVersion] = GetParam(); + const auto expr = fmt("builtins.parseDrvName \"%1%\"", input); + auto v = eval(expr); + ASSERT_THAT(v, IsAttrsOfSize(2)); + + auto name = v.attrs->find(createSymbol("name")); + ASSERT_TRUE(name); + ASSERT_THAT(*name->value, IsStringEq(expectedName)); + + auto version = v.attrs->find(createSymbol("version")); + ASSERT_TRUE(version); + ASSERT_THAT(*version->value, IsStringEq(expectedVersion)); + } + + INSTANTIATE_TEST_SUITE_P( + parseDrvName, + ParseDrvNamePrimOpTest, + testing::Values( + std::make_tuple("nix-0.12pre12876", "nix", "0.12pre12876"), + std::make_tuple("a-b-c-1234pre5+git", "a-b-c", "1234pre5+git") + ) + ); + + TEST_F(PrimOpTest, replaceStrings) { + // FIXME: add a test that verifies the string context is as expected + auto v = eval("builtins.replaceStrings [\"oo\" \"a\"] [\"a\" \"i\"] \"foobar\""); + ASSERT_EQ(v.type(), nString); + ASSERT_EQ(v.string.s, std::string_view("fabir")); + } + + TEST_F(PrimOpTest, concatStringsSep) { + // FIXME: add a test that verifies the string context is as expected + auto v = eval("builtins.concatStringsSep \"%\" [\"foo\" \"bar\" \"baz\"]"); + ASSERT_EQ(v.type(), nString); + ASSERT_EQ(std::string_view(v.string.s), "foo%bar%baz"); + } + + TEST_F(PrimOpTest, split1) { + // v = [ "" [ "a" ] "c" ] + auto v = eval("builtins.split \"(a)b\" \"abc\""); + ASSERT_THAT(v, IsListOfSize(3)); + + ASSERT_THAT(*v.listElems()[0], IsStringEq("")); + + ASSERT_THAT(*v.listElems()[1], IsListOfSize(1)); + ASSERT_THAT(*v.listElems()[1]->listElems()[0], IsStringEq("a")); + + ASSERT_THAT(*v.listElems()[2], IsStringEq("c")); + } + + TEST_F(PrimOpTest, split2) { + // v is expected to be a list [ "" [ "a" ] "b" [ "c"] "" ] + auto v = eval("builtins.split \"([ac])\" \"abc\""); + ASSERT_THAT(v, IsListOfSize(5)); + + ASSERT_THAT(*v.listElems()[0], IsStringEq("")); + + ASSERT_THAT(*v.listElems()[1], IsListOfSize(1)); + ASSERT_THAT(*v.listElems()[1]->listElems()[0], IsStringEq("a")); + + ASSERT_THAT(*v.listElems()[2], IsStringEq("b")); + + ASSERT_THAT(*v.listElems()[3], IsListOfSize(1)); + ASSERT_THAT(*v.listElems()[3]->listElems()[0], IsStringEq("c")); + + ASSERT_THAT(*v.listElems()[4], IsStringEq("")); + } + + TEST_F(PrimOpTest, split3) { + auto v = eval("builtins.split \"(a)|(c)\" \"abc\""); + ASSERT_THAT(v, IsListOfSize(5)); + + // First list element + ASSERT_THAT(*v.listElems()[0], IsStringEq("")); + + // 2nd list element is a list [ "" null ] + ASSERT_THAT(*v.listElems()[1], IsListOfSize(2)); + ASSERT_THAT(*v.listElems()[1]->listElems()[0], IsStringEq("a")); + ASSERT_THAT(*v.listElems()[1]->listElems()[1], IsNull()); + + // 3rd element + ASSERT_THAT(*v.listElems()[2], IsStringEq("b")); + + // 4th element is a list: [ null "c" ] + ASSERT_THAT(*v.listElems()[3], IsListOfSize(2)); + ASSERT_THAT(*v.listElems()[3]->listElems()[0], IsNull()); + ASSERT_THAT(*v.listElems()[3]->listElems()[1], IsStringEq("c")); + + // 5th element is the empty string + ASSERT_THAT(*v.listElems()[4], IsStringEq("")); + } + + TEST_F(PrimOpTest, split4) { + auto v = eval("builtins.split \"([[:upper:]]+)\" \" FOO \""); + ASSERT_THAT(v, IsListOfSize(3)); + auto first = v.listElems()[0]; + auto second = v.listElems()[1]; + auto third = v.listElems()[2]; + + ASSERT_THAT(*first, IsStringEq(" ")); + + ASSERT_THAT(*second, IsListOfSize(1)); + ASSERT_THAT(*second->listElems()[0], IsStringEq("FOO")); + + ASSERT_THAT(*third, IsStringEq(" ")); + } + + TEST_F(PrimOpTest, match1) { + auto v = eval("builtins.match \"ab\" \"abc\""); + ASSERT_THAT(v, IsNull()); + } + + TEST_F(PrimOpTest, match2) { + auto v = eval("builtins.match \"abc\" \"abc\""); + ASSERT_THAT(v, IsListOfSize(0)); + } + + TEST_F(PrimOpTest, match3) { + auto v = eval("builtins.match \"a(b)(c)\" \"abc\""); + ASSERT_THAT(v, IsListOfSize(2)); + ASSERT_THAT(*v.listElems()[0], IsStringEq("b")); + ASSERT_THAT(*v.listElems()[1], IsStringEq("c")); + } + + TEST_F(PrimOpTest, match4) { + auto v = eval("builtins.match \"[[:space:]]+([[:upper:]]+)[[:space:]]+\" \" FOO \""); + ASSERT_THAT(v, IsListOfSize(1)); + ASSERT_THAT(*v.listElems()[0], IsStringEq("FOO")); + } + + TEST_F(PrimOpTest, attrNames) { + auto v = eval("builtins.attrNames { x = 1; y = 2; z = 3; a = 2; }"); + ASSERT_THAT(v, IsListOfSize(4)); + + // ensure that the list is sorted + const std::vector expected { "a", "x", "y", "z" }; + for (const auto [n, elem] : enumerate(v.listItems())) + ASSERT_THAT(*elem, IsStringEq(expected[n])); + } + + TEST_F(PrimOpTest, genericClosure_not_strict) { + // Operator should not be used when startSet is empty + auto v = eval("builtins.genericClosure { startSet = []; }"); + ASSERT_THAT(v, IsListOfSize(0)); + } +} /* namespace nix */ diff -Nru nix-2.8.0/src/libexpr/tests/trivial.cc nix-2.16.1+dfsg/src/libexpr/tests/trivial.cc --- nix-2.8.0/src/libexpr/tests/trivial.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/tests/trivial.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,196 @@ +#include "tests/libexpr.hh" + +namespace nix { + // Testing of trivial expressions + class TrivialExpressionTest : public LibExprTest {}; + + TEST_F(TrivialExpressionTest, true) { + auto v = eval("true"); + ASSERT_THAT(v, IsTrue()); + } + + TEST_F(TrivialExpressionTest, false) { + auto v = eval("false"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, null) { + auto v = eval("null"); + ASSERT_THAT(v, IsNull()); + } + + TEST_F(TrivialExpressionTest, 1) { + auto v = eval("1"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, 1plus1) { + auto v = eval("1+1"); + ASSERT_THAT(v, IsIntEq(2)); + } + + TEST_F(TrivialExpressionTest, minus1) { + auto v = eval("-1"); + ASSERT_THAT(v, IsIntEq(-1)); + } + + TEST_F(TrivialExpressionTest, 1minus1) { + auto v = eval("1-1"); + ASSERT_THAT(v, IsIntEq(0)); + } + + TEST_F(TrivialExpressionTest, lambdaAdd) { + auto v = eval("let add = a: b: a + b; in add 1 2"); + ASSERT_THAT(v, IsIntEq(3)); + } + + TEST_F(TrivialExpressionTest, list) { + auto v = eval("[]"); + ASSERT_THAT(v, IsListOfSize(0)); + } + + TEST_F(TrivialExpressionTest, attrs) { + auto v = eval("{}"); + ASSERT_THAT(v, IsAttrsOfSize(0)); + } + + TEST_F(TrivialExpressionTest, float) { + auto v = eval("1.234"); + ASSERT_THAT(v, IsFloatEq(1.234)); + } + + TEST_F(TrivialExpressionTest, updateAttrs) { + auto v = eval("{ a = 1; } // { b = 2; a = 3; }"); + ASSERT_THAT(v, IsAttrsOfSize(2)); + auto a = v.attrs->find(createSymbol("a")); + ASSERT_NE(a, nullptr); + ASSERT_THAT(*a->value, IsIntEq(3)); + + auto b = v.attrs->find(createSymbol("b")); + ASSERT_NE(b, nullptr); + ASSERT_THAT(*b->value, IsIntEq(2)); + } + + TEST_F(TrivialExpressionTest, hasAttrOpFalse) { + auto v = eval("{} ? a"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, hasAttrOpTrue) { + auto v = eval("{ a = 123; } ? a"); + ASSERT_THAT(v, IsTrue()); + } + + TEST_F(TrivialExpressionTest, withFound) { + auto v = eval("with { a = 23; }; a"); + ASSERT_THAT(v, IsIntEq(23)); + } + + TEST_F(TrivialExpressionTest, withNotFound) { + ASSERT_THROW(eval("with {}; a"), Error); + } + + TEST_F(TrivialExpressionTest, withOverride) { + auto v = eval("with { a = 23; }; with { a = 42; }; a"); + ASSERT_THAT(v, IsIntEq(42)); + } + + TEST_F(TrivialExpressionTest, letOverWith) { + auto v = eval("let a = 23; in with { a = 1; }; a"); + ASSERT_THAT(v, IsIntEq(23)); + } + + TEST_F(TrivialExpressionTest, multipleLet) { + auto v = eval("let a = 23; in let a = 42; in a"); + ASSERT_THAT(v, IsIntEq(42)); + } + + TEST_F(TrivialExpressionTest, defaultFunctionArgs) { + auto v = eval("({ a ? 123 }: a) {}"); + ASSERT_THAT(v, IsIntEq(123)); + } + + TEST_F(TrivialExpressionTest, defaultFunctionArgsOverride) { + auto v = eval("({ a ? 123 }: a) { a = 5; }"); + ASSERT_THAT(v, IsIntEq(5)); + } + + TEST_F(TrivialExpressionTest, defaultFunctionArgsCaptureBack) { + auto v = eval("({ a ? 123 }@args: args) {}"); + ASSERT_THAT(v, IsAttrsOfSize(0)); + } + + TEST_F(TrivialExpressionTest, defaultFunctionArgsCaptureFront) { + auto v = eval("(args@{ a ? 123 }: args) {}"); + ASSERT_THAT(v, IsAttrsOfSize(0)); + } + + TEST_F(TrivialExpressionTest, assertThrows) { + ASSERT_THROW(eval("let x = arg: assert arg == 1; 123; in x 2"), Error); + } + + TEST_F(TrivialExpressionTest, assertPassed) { + auto v = eval("let x = arg: assert arg == 1; 123; in x 1"); + ASSERT_THAT(v, IsIntEq(123)); + } + + class AttrSetMergeTrvialExpressionTest : + public TrivialExpressionTest, + public testing::WithParamInterface + {}; + + TEST_P(AttrSetMergeTrvialExpressionTest, attrsetMergeLazy) { + // Usually Nix rejects duplicate keys in an attrset but it does allow + // so if it is an attribute set that contains disjoint sets of keys. + // The below is equivalent to `{a.b = 1; a.c = 2; }`. + // The attribute set `a` will be a Thunk at first as the attribuets + // have to be merged (or otherwise computed) and that is done in a lazy + // manner. + + auto expr = GetParam(); + auto v = eval(expr); + ASSERT_THAT(v, IsAttrsOfSize(1)); + + auto a = v.attrs->find(createSymbol("a")); + ASSERT_NE(a, nullptr); + + ASSERT_THAT(*a->value, IsThunk()); + state.forceValue(*a->value, noPos); + + ASSERT_THAT(*a->value, IsAttrsOfSize(2)); + + auto b = a->value->attrs->find(createSymbol("b")); + ASSERT_NE(b, nullptr); + ASSERT_THAT(*b->value, IsIntEq(1)); + + auto c = a->value->attrs->find(createSymbol("c")); + ASSERT_NE(c, nullptr); + ASSERT_THAT(*c->value, IsIntEq(2)); + } + + INSTANTIATE_TEST_SUITE_P( + attrsetMergeLazy, + AttrSetMergeTrvialExpressionTest, + testing::Values( + "{ a.b = 1; a.c = 2; }", + "{ a = { b = 1; }; a = { c = 2; }; }" + ) + ); + + TEST_F(TrivialExpressionTest, functor) { + auto v = eval("{ __functor = self: arg: self.v + arg; v = 10; } 5"); + ASSERT_THAT(v, IsIntEq(15)); + } + + TEST_F(TrivialExpressionTest, bindOr) { + auto v = eval("{ or = 1; }"); + ASSERT_THAT(v, IsAttrsOfSize(1)); + auto b = v.attrs->find(createSymbol("or")); + ASSERT_NE(b, nullptr); + ASSERT_THAT(*b->value, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, orCantBeUsed) { + ASSERT_THROW(eval("let or = 1; in or"), Error); + } +} /* namespace nix */ diff -Nru nix-2.8.0/src/libexpr/tests/value/context.cc nix-2.16.1+dfsg/src/libexpr/tests/value/context.cc --- nix-2.8.0/src/libexpr/tests/value/context.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/tests/value/context.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,122 @@ +#include +#include +#include + +#include "tests/path.hh" +#include "tests/libexpr.hh" +#include "tests/value/context.hh" + +namespace nix { + +TEST(NixStringContextElemTest, empty_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(""), + BadNixStringContextElem); +} + +TEST(NixStringContextElemTest, single_bang_invalid) { + EXPECT_THROW( + NixStringContextElem::parse("!"), + BadNixStringContextElem); +} + +TEST(NixStringContextElemTest, double_bang_invalid) { + EXPECT_THROW( + NixStringContextElem::parse("!!/"), + BadStorePath); +} + +TEST(NixStringContextElemTest, eq_slash_invalid) { + EXPECT_THROW( + NixStringContextElem::parse("=/"), + BadStorePath); +} + +TEST(NixStringContextElemTest, slash_invalid) { + EXPECT_THROW( + NixStringContextElem::parse("/"), + BadStorePath); +} + +TEST(NixStringContextElemTest, opaque) { + std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = NixStringContextElem::parse(opaque); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->path, StorePath { opaque }); + ASSERT_EQ(elem.to_string(), opaque); +} + +TEST(NixStringContextElemTest, drvDeep) { + std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(drvDeep); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->drvPath, StorePath { drvDeep.substr(1) }); + ASSERT_EQ(elem.to_string(), drvDeep); +} + +TEST(NixStringContextElemTest, built) { + std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->output, "foo"); + ASSERT_EQ(p->drvPath, StorePath { built.substr(5) }); + ASSERT_EQ(elem.to_string(), built); +} + +} + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::Opaque { + .path = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::Built { + .drvPath = *gen::arbitrary(), + .output = (*gen::arbitrary()).name, + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + case 2: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} + +namespace nix { + +RC_GTEST_PROP( + NixStringContextElemTest, + prop_round_rip, + (const NixStringContextElem & o)) +{ + RC_ASSERT(o == NixStringContextElem::parse(o.to_string())); +} + +} diff -Nru nix-2.8.0/src/libexpr/tests/value/context.hh nix-2.16.1+dfsg/src/libexpr/tests/value/context.hh --- nix-2.8.0/src/libexpr/tests/value/context.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/tests/value/context.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include + +#include + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff -Nru nix-2.8.0/src/libexpr/value/context.cc nix-2.16.1+dfsg/src/libexpr/value/context.cc --- nix-2.8.0/src/libexpr/value/context.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/value/context.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,66 @@ +#include "value/context.hh" + +#include + +namespace nix { + +NixStringContextElem NixStringContextElem::parse(std::string_view s0) +{ + std::string_view s = s0; + + if (s.size() == 0) { + throw BadNixStringContextElem(s0, + "String context element should never be an empty string"); + } + switch (s.at(0)) { + case '!': { + s = s.substr(1); // advance string to parse after first ! + size_t index = s.find("!"); + // This makes index + 1 safe. Index can be the length (one after index + // of last character), so given any valid character index --- a + // successful find --- we can add one. + if (index == std::string_view::npos) { + throw BadNixStringContextElem(s0, + "String content element beginning with '!' should have a second '!'"); + } + return NixStringContextElem::Built { + .drvPath = StorePath { s.substr(index + 1) }, + .output = std::string(s.substr(0, index)), + }; + } + case '=': { + return NixStringContextElem::DrvDeep { + .drvPath = StorePath { s.substr(1) }, + }; + } + default: { + return NixStringContextElem::Opaque { + .path = StorePath { s }, + }; + } + } +} + +std::string NixStringContextElem::to_string() const { + return std::visit(overloaded { + [&](const NixStringContextElem::Built & b) { + std::string res; + res += '!'; + res += b.output; + res += '!'; + res += b.drvPath.to_string(); + return res; + }, + [&](const NixStringContextElem::DrvDeep & d) { + std::string res; + res += '='; + res += d.drvPath.to_string(); + return res; + }, + [&](const NixStringContextElem::Opaque & o) { + return std::string { o.path.to_string() }; + }, + }, raw()); +} + +} diff -Nru nix-2.8.0/src/libexpr/value/context.hh nix-2.16.1+dfsg/src/libexpr/value/context.hh --- nix-2.8.0/src/libexpr/value/context.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/value/context.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,103 @@ +#pragma once +///@file + +#include "util.hh" +#include "comparator.hh" +#include "path.hh" + +#include + +#include + +namespace nix { + +class BadNixStringContextElem : public Error +{ +public: + std::string_view raw; + + template + BadNixStringContextElem(std::string_view raw_, const Args & ... args) + : Error("") + { + raw = raw_; + auto hf = hintfmt(args...); + err.msg = hintfmt("Bad String Context element: %1%: %2%", normaltxt(hf.str()), raw); + } +}; + +/** + * Plain opaque path to some store object. + * + * Encoded as just the path: ‘’. + */ +struct NixStringContextElem_Opaque { + StorePath path; + + GENERATE_CMP(NixStringContextElem_Opaque, me->path); +}; + +/** + * Path to a derivation and its entire build closure. + * + * The path doesn't just refer to derivation itself and its closure, but + * also all outputs of all derivations in that closure (including the + * root derivation). + * + * Encoded in the form ‘=’. + */ +struct NixStringContextElem_DrvDeep { + StorePath drvPath; + + GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath); +}; + +/** + * Derivation output. + * + * Encoded in the form ‘!!’. + */ +struct NixStringContextElem_Built { + StorePath drvPath; + std::string output; + + GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output); +}; + +using _NixStringContextElem_Raw = std::variant< + NixStringContextElem_Opaque, + NixStringContextElem_DrvDeep, + NixStringContextElem_Built +>; + +struct NixStringContextElem : _NixStringContextElem_Raw { + using Raw = _NixStringContextElem_Raw; + using Raw::Raw; + + using Opaque = NixStringContextElem_Opaque; + using DrvDeep = NixStringContextElem_DrvDeep; + using Built = NixStringContextElem_Built; + + inline const Raw & raw() const & { + return static_cast(*this); + } + inline Raw & raw() & { + return static_cast(*this); + } + inline Raw && raw() && { + return static_cast(*this); + } + + /** + * Decode a context string, one of: + * - ‘’ + * - ‘=’ + * - ‘!!’ + */ + static NixStringContextElem parse(std::string_view s); + std::string to_string() const; +}; + +typedef std::set NixStringContext; + +} diff -Nru nix-2.8.0/src/libexpr/value.hh nix-2.16.1+dfsg/src/libexpr/value.hh --- nix-2.8.0/src/libexpr/value.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/value.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,12 +1,16 @@ #pragma once +///@file #include #include "symbol-table.hh" +#include "value/context.hh" +#include "input-accessor.hh" #if HAVE_BOEHMGC #include #endif +#include namespace nix { @@ -33,9 +37,11 @@ tFloat } InternalType; -// This type abstracts over all actual value types in the language, -// grouping together implementation details like tList*, different function -// types, and types in non-normal form (so thunks and co.) +/** + * This type abstracts over all actual value types in the language, + * grouping together implementation details like tList*, different function + * types, and types in non-normal form (so thunks and co.) + */ typedef enum { nThunk, nInt, @@ -56,54 +62,65 @@ struct ExprLambda; struct PrimOp; class Symbol; +class PosIdx; struct Pos; class StorePath; class Store; class EvalState; class XMLWriter; -class JSONPlaceholder; typedef int64_t NixInt; typedef double NixFloat; -typedef std::pair NixStringContextElem; -typedef std::vector NixStringContext; -/* External values must descend from ExternalValueBase, so that +/** + * External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented */ class ExternalValueBase { friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); protected: - /* Print out the value */ + /** + * Print out the value + */ virtual std::ostream & print(std::ostream & str) const = 0; public: - /* Return a simple string describing the type */ + /** + * Return a simple string describing the type + */ virtual std::string showType() const = 0; - /* Return a string to be used in builtins.typeOf */ + /** + * Return a string to be used in builtins.typeOf + */ virtual std::string typeOf() const = 0; - /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an + /** + * Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error. */ - virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; + virtual std::string coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const; - /* Compare to another value of the same type. Defaults to uncomparable, + /** + * Compare to another value of the same type. Defaults to uncomparable, * i.e. always false. */ virtual bool operator ==(const ExternalValueBase & b) const; - /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ - virtual void printValueAsJSON(EvalState & state, bool strict, - JSONPlaceholder & out, PathSet & context) const; + /** + * Print the value as JSON. Defaults to unconvertable, i.e. throws an error + */ + virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict, + NixStringContext & context, bool copyToStore = true) const; - /* Print the value as XML. Defaults to unevaluated */ + /** + * Print the value as XML. Defaults to unevaluated + */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, - XMLWriter & doc, PathSet & context, PathSet & drvsSeen, - const Pos & pos) const; + XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, + const PosIdx pos) const; virtual ~ExternalValueBase() { @@ -120,11 +137,11 @@ friend std::string showType(const Value & v); - void print(std::ostream & str, std::set * seen) const; + void print(const SymbolTable & symbols, std::ostream & str, std::set * seen) const; public: - void print(std::ostream & str, bool showRepeated = false) const; + void print(const SymbolTable & symbols, std::ostream & str, bool showRepeated = false) const; // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's @@ -145,32 +162,34 @@ NixInt integer; bool boolean; - /* Strings in the evaluator carry a so-called `context' which - is a list of strings representing store paths. This is to - allow users to write things like - - "--with-freetype2-library=" + freetype + "/lib" - - where `freetype' is a derivation (or a source to be copied - to the store). If we just concatenated the strings without - keeping track of the referenced store paths, then if the - string is used as a derivation attribute, the derivation - will not have the correct dependencies in its inputDrvs and - inputSrcs. - - The semantics of the context is as follows: when a string - with context C is used as a derivation attribute, then the - derivations in C will be added to the inputDrvs of the - derivation, and the other store paths in C will be added to - the inputSrcs of the derivations. + /** + * Strings in the evaluator carry a so-called `context` which + * is a list of strings representing store paths. This is to + * allow users to write things like + + * "--with-freetype2-library=" + freetype + "/lib" + + * where `freetype` is a derivation (or a source to be copied + * to the store). If we just concatenated the strings without + * keeping track of the referenced store paths, then if the + * string is used as a derivation attribute, the derivation + * will not have the correct dependencies in its inputDrvs and + * inputSrcs. + + * The semantics of the context is as follows: when a string + * with context C is used as a derivation attribute, then the + * derivations in C will be added to the inputDrvs of the + * derivation, and the other store paths in C will be added to + * the inputSrcs of the derivations. - For canonicity, the store paths should be in sorted order. */ + * For canonicity, the store paths should be in sorted order. + */ struct { const char * s; const char * * context; // must be in sorted order } string; - const char * path; + const char * _path; Bindings * attrs; struct { size_t size; @@ -196,8 +215,10 @@ NixFloat fpoint; }; - // Returns the normal type of a Value. This only returns nThunk if the - // Value hasn't been forceValue'd + /** + * Returns the normal type of a Value. This only returns nThunk if + * the Value hasn't been forceValue'd + */ inline ValueType type() const { switch (internalType) { @@ -216,8 +237,10 @@ abort(); } - /* After overwriting an app node, be sure to clear pointers in the - Value to ensure that the target isn't kept alive unnecessarily. */ + /** + * After overwriting an app node, be sure to clear pointers in the + * Value to ensure that the target isn't kept alive unnecessarily. + */ inline void clearValue() { app.left = app.right = 0; @@ -246,24 +269,24 @@ void mkString(std::string_view s); - void mkString(std::string_view s, const PathSet & context); + void mkString(std::string_view s, const NixStringContext & context); - void mkStringMove(const char * s, const PathSet & context); + void mkStringMove(const char * s, const NixStringContext & context); inline void mkString(const Symbol & s) { mkString(((const std::string &) s).c_str()); } - inline void mkPath(const char * s) + void mkPath(const SourcePath & path); + + inline void mkPath(const char * path) { clearValue(); internalType = tPath; - path = s; + _path = path; } - void mkPath(std::string_view s); - inline void mkNull() { clearValue(); @@ -368,15 +391,15 @@ return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size; } - Pos determinePos(const Pos & pos) const; + PosIdx determinePos(const PosIdx pos) const; - /* Check whether forcing this value requires a trivial amount of - computation. In particular, function applications are - non-trivial. */ + /** + * Check whether forcing this value requires a trivial amount of + * computation. In particular, function applications are + * non-trivial. + */ bool isTrivial() const; - NixStringContext getContext(const Store &); - auto listItems() { struct ListIterable @@ -404,13 +427,25 @@ auto begin = listElems(); return ConstListIterable { begin, begin + listSize() }; } + + SourcePath path() const + { + assert(internalType == tPath); + return SourcePath{CanonPath(_path)}; + } + + std::string_view str() const + { + assert(internalType == tString); + return std::string_view(string.s); + } }; #if HAVE_BOEHMGC -typedef std::vector > ValueVector; -typedef std::map, traceable_allocator > > ValueMap; -typedef std::map, traceable_allocator > > ValueVectorMap; +typedef std::vector> ValueVector; +typedef std::map, traceable_allocator>> ValueMap; +typedef std::map, traceable_allocator>> ValueVectorMap; #else typedef std::vector ValueVector; typedef std::map ValueMap; @@ -418,7 +453,9 @@ #endif -/* A value allocated in traceable memory. */ +/** + * A value allocated in traceable memory. + */ typedef std::shared_ptr RootValue; RootValue allocRootValue(Value * v); diff -Nru nix-2.8.0/src/libexpr/value-to-json.cc nix-2.16.1+dfsg/src/libexpr/value-to-json.cc --- nix-2.8.0/src/libexpr/value-to-json.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/value-to-json.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,105 +1,109 @@ #include "value-to-json.hh" -#include "json.hh" #include "eval-inline.hh" #include "util.hh" +#include "store-api.hh" #include #include +#include namespace nix { - -void printValueAsJSON(EvalState & state, bool strict, - Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context) +using json = nlohmann::json; +json printValueAsJSON(EvalState & state, bool strict, + Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore) { checkInterrupt(); if (strict) state.forceValue(v, pos); + json out; + switch (v.type()) { case nInt: - out.write(v.integer); + out = v.integer; break; case nBool: - out.write(v.boolean); + out = v.boolean; break; case nString: copyContext(v, context); - out.write(v.string.s); + out = v.string.s; break; case nPath: - out.write(state.copyPathToStore(context, v.path)); + if (copyToStore) + out = state.store->printStorePath( + state.copyPathToStore(context, v.path())); + else + out = v.path().path.abs(); break; case nNull: - out.write(nullptr); break; case nAttrs: { auto maybeString = state.tryAttrsToString(pos, v, context, false, false); if (maybeString) { - out.write(*maybeString); + out = *maybeString; break; } auto i = v.attrs->find(state.sOutPath); if (i == v.attrs->end()) { - auto obj(out.object()); + out = json::object(); StringSet names; for (auto & j : *v.attrs) - names.insert(j.name); + names.emplace(state.symbols[j.name]); for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); - auto placeholder(obj.placeholder(j)); - printValueAsJSON(state, strict, *a.value, *a.pos, placeholder, context); + out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); } } else - printValueAsJSON(state, strict, *i->value, *i->pos, out, context); + return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore); break; } case nList: { - auto list(out.list()); - for (auto elem : v.listItems()) { - auto placeholder(list.placeholder()); - printValueAsJSON(state, strict, *elem, pos, placeholder, context); - } + out = json::array(); + for (auto elem : v.listItems()) + out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); break; } case nExternal: - v.external->printValueAsJSON(state, strict, out, context); + return v.external->printValueAsJSON(state, strict, context, copyToStore); break; case nFloat: - out.write(v.fpoint); + out = v.fpoint; break; case nThunk: case nFunction: auto e = TypeError({ .msg = hintfmt("cannot convert %1% to JSON", showType(v)), - .errPos = v.determinePos(pos) + .errPos = state.positions[v.determinePos(pos)] }); - e.addTrace(pos, hintfmt("message for the trace")); + e.addTrace(state.positions[pos], hintfmt("message for the trace")); + state.debugThrowLastTrace(e); throw e; } + return out; } void printValueAsJSON(EvalState & state, bool strict, - Value & v, const Pos & pos, std::ostream & str, PathSet & context) + Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore) { - JSONPlaceholder out(str); - printValueAsJSON(state, strict, v, pos, out, context); + str << printValueAsJSON(state, strict, v, pos, context, copyToStore); } -void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, - JSONPlaceholder & out, PathSet & context) const +json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, + NixStringContext & context, bool copyToStore) const { - throw TypeError("cannot convert %1% to JSON", showType()); + state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType())); } diff -Nru nix-2.8.0/src/libexpr/value-to-json.hh nix-2.16.1+dfsg/src/libexpr/value-to-json.hh --- nix-2.8.0/src/libexpr/value-to-json.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/value-to-json.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,19 +1,19 @@ #pragma once +///@file #include "nixexpr.hh" #include "eval.hh" #include #include +#include namespace nix { -class JSONPlaceholder; +nlohmann::json printValueAsJSON(EvalState & state, bool strict, + Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore = true); void printValueAsJSON(EvalState & state, bool strict, - Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context); - -void printValueAsJSON(EvalState & state, bool strict, - Value & v, const Pos & pos, std::ostream & str, PathSet & context); + Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore = true); } diff -Nru nix-2.8.0/src/libexpr/value-to-xml.cc nix-2.16.1+dfsg/src/libexpr/value-to-xml.cc --- nix-2.8.0/src/libexpr/value-to-xml.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/value-to-xml.cc 2023-06-06 10:55:23.000000000 +0000 @@ -18,43 +18,44 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, - const Pos & pos); + Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, + const PosIdx pos); -static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos) +static void posToXML(EvalState & state, XMLAttrs & xmlAttrs, const Pos & pos) { - xmlAttrs["path"] = pos.file; - xmlAttrs["line"] = (format("%1%") % pos.line).str(); - xmlAttrs["column"] = (format("%1%") % pos.column).str(); + if (auto path = std::get_if(&pos.origin)) + xmlAttrs["path"] = path->path.abs(); + xmlAttrs["line"] = fmt("%1%", pos.line); + xmlAttrs["column"] = fmt("%1%", pos.column); } static void showAttrs(EvalState & state, bool strict, bool location, - Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) + Bindings & attrs, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen) { StringSet names; for (auto & i : attrs) - names.insert(i.name); + names.emplace(state.symbols[i.name]); for (auto & i : names) { Attr & a(*attrs.find(state.symbols.create(i))); XMLAttrs xmlAttrs; xmlAttrs["name"] = i; - if (location && a.pos != ptr(&noPos)) posToXML(xmlAttrs, *a.pos); + if (location && a.pos) posToXML(state, xmlAttrs, state.positions[a.pos]); XMLOpenElement _(doc, "attr", xmlAttrs); printValueAsXML(state, strict, location, - *a.value, doc, context, drvsSeen, *a.pos); + *a.value, doc, context, drvsSeen, a.pos); } } static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, - const Pos & pos) + Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, + const PosIdx pos) { checkInterrupt(); @@ -63,7 +64,7 @@ switch (v.type()) { case nInt: - doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); + doc.writeEmptyElement("int", singletonAttrs("value", fmt("%1%", v.integer))); break; case nBool: @@ -77,7 +78,7 @@ break; case nPath: - doc.writeEmptyElement("path", singletonAttrs("value", v.path)); + doc.writeEmptyElement("path", singletonAttrs("value", v.path().to_string())); break; case nNull: @@ -93,14 +94,14 @@ Path drvPath; a = v.attrs->find(state.sDrvPath); if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value, *a->pos); + if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) xmlAttrs["drvPath"] = drvPath = a->value->string.s; } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value, *a->pos); + if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) xmlAttrs["outPath"] = a->value->string.s; } @@ -134,18 +135,18 @@ break; } XMLAttrs xmlAttrs; - if (location) posToXML(xmlAttrs, v.lambda.fun->pos); + if (location) posToXML(state, xmlAttrs, state.positions[v.lambda.fun->pos]); XMLOpenElement _(doc, "function", xmlAttrs); if (v.lambda.fun->hasFormals()) { XMLAttrs attrs; - if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg; + if (v.lambda.fun->arg) attrs["name"] = state.symbols[v.lambda.fun->arg]; if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; XMLOpenElement _(doc, "attrspat", attrs); - for (auto & i : v.lambda.fun->formals->lexicographicOrder()) - doc.writeEmptyElement("attr", singletonAttrs("name", i.name)); + for (auto & i : v.lambda.fun->formals->lexicographicOrder(state.symbols)) + doc.writeEmptyElement("attr", singletonAttrs("name", state.symbols[i.name])); } else - doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg)); + doc.writeEmptyElement("varpat", singletonAttrs("name", state.symbols[v.lambda.fun->arg])); break; } @@ -155,7 +156,7 @@ break; case nFloat: - doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); + doc.writeEmptyElement("float", singletonAttrs("value", fmt("%1%", v.fpoint))); break; case nThunk: @@ -165,15 +166,15 @@ void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, - bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, - const Pos & pos) const + bool location, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, + const PosIdx pos) const { doc.writeEmptyElement("unevaluated"); } void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context, const Pos & pos) + Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos) { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); diff -Nru nix-2.8.0/src/libexpr/value-to-xml.hh nix-2.16.1+dfsg/src/libexpr/value-to-xml.hh --- nix-2.8.0/src/libexpr/value-to-xml.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libexpr/value-to-xml.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "nixexpr.hh" #include "eval.hh" @@ -9,6 +10,6 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context, const Pos & pos); + Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos); } diff -Nru nix-2.8.0/src/libfetchers/attrs.hh nix-2.16.1+dfsg/src/libfetchers/attrs.hh --- nix-2.8.0/src/libfetchers/attrs.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/attrs.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff -Nru nix-2.8.0/src/libfetchers/cache.hh nix-2.16.1+dfsg/src/libfetchers/cache.hh --- nix-2.8.0/src/libfetchers/cache.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/cache.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "fetchers.hh" diff -Nru nix-2.8.0/src/libfetchers/fetchers.cc nix-2.16.1+dfsg/src/libfetchers/fetchers.cc --- nix-2.8.0/src/libfetchers/fetchers.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/fetchers.cc 2023-06-06 10:55:23.000000000 +0000 @@ -210,7 +210,13 @@ auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName()); + return store.makeFixedOutputPath(getName(), FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Recursive, + .hash = *narHash, + }, + .references = {}, + }); } std::string Input::getType() const @@ -266,7 +272,7 @@ return {}; } -ParsedURL InputScheme::toURL(const Input & input) +ParsedURL InputScheme::toURL(const Input & input) const { throw Error("don't know how to convert input '%s' to a URL", attrsToJSON(input.attrs)); } @@ -274,7 +280,7 @@ Input InputScheme::applyOverrides( const Input & input, std::optional ref, - std::optional rev) + std::optional rev) const { if (ref) throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref); @@ -293,7 +299,7 @@ assert(false); } -void InputScheme::clone(const Input & input, const Path & destDir) +void InputScheme::clone(const Input & input, const Path & destDir) const { throw Error("do not know how to clone input '%s'", input.to_string()); } diff -Nru nix-2.8.0/src/libfetchers/fetchers.hh nix-2.16.1+dfsg/src/libfetchers/fetchers.hh --- nix-2.8.0/src/libfetchers/fetchers.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/fetchers.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" @@ -20,14 +21,14 @@ struct InputScheme; -/* The Input object is generated by a specific fetcher, based on the +/** + * The Input object is generated by a specific fetcher, based on the * user-supplied input attribute in the flake.nix file, and contains * the information that the specific fetcher needs to perform the * actual fetch. The Input object is most commonly created via the * "fromURL()" or "fromAttrs()" static functions which are provided * the url or attrset specified in the flake file. */ - struct Input { friend struct InputScheme; @@ -37,7 +38,9 @@ bool locked = false; bool direct = true; - /* path of the parent of this input, used for relative path resolution */ + /** + * path of the parent of this input, used for relative path resolution + */ std::optional parent; public: @@ -55,22 +58,35 @@ Attrs toAttrs() const; - /* Check whether this is a "direct" input, that is, not - one that goes through a registry. */ + /** + * Check whether this is a "direct" input, that is, not + * one that goes through a registry. + */ bool isDirect() const { return direct; } - /* Check whether this is a "locked" input, that is, - one that contains a commit hash or content hash. */ + /** + * Check whether this is a "locked" input, that is, + * one that contains a commit hash or content hash. + */ bool isLocked() const { return locked; } + /** + * Check whether the input carries all necessary info required + * for cache insertion and substitution. + * These fields are used to uniquely identify cached trees + * within the "tarball TTL" window without necessarily + * indicating that the input's origin is unchanged. + */ bool hasAllInfo() const; bool operator ==(const Input & other) const; bool contains(const Input & other) const; - /* Fetch the input into the Nix store, returning the location in - the Nix store and the locked input. */ + /** + * Fetch the input into the Nix store, returning the location in + * the Nix store and the locked input. + */ std::pair fetch(ref store) const; Input applyOverrides( @@ -99,7 +115,8 @@ }; -/* The InputScheme represents a type of fetcher. Each fetcher +/** + * The InputScheme represents a type of fetcher. Each fetcher * registers with nix at startup time. When processing an input for a * flake, each scheme is given an opportunity to "recognize" that * input from the url or attributes in the flake file's specification @@ -107,26 +124,25 @@ * recognized. The Input object contains the information the fetcher * needs to actually perform the "fetch()" when called. */ - struct InputScheme { virtual ~InputScheme() { } - virtual std::optional inputFromURL(const ParsedURL & url) = 0; + virtual std::optional inputFromURL(const ParsedURL & url) const = 0; - virtual std::optional inputFromAttrs(const Attrs & attrs) = 0; + virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; - virtual ParsedURL toURL(const Input & input); + virtual ParsedURL toURL(const Input & input) const; - virtual bool hasAllInfo(const Input & input) = 0; + virtual bool hasAllInfo(const Input & input) const = 0; virtual Input applyOverrides( const Input & input, std::optional ref, - std::optional rev); + std::optional rev) const; - virtual void clone(const Input & input, const Path & destDir); + virtual void clone(const Input & input, const Path & destDir) const; virtual std::optional getSourcePath(const Input & input); diff -Nru nix-2.8.0/src/libfetchers/fetch-settings.hh nix-2.16.1+dfsg/src/libfetchers/fetch-settings.hh --- nix-2.8.0/src/libfetchers/fetch-settings.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/fetch-settings.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "config.hh" @@ -57,7 +58,7 @@ ``` This example specifies three tokens, one each for accessing - github.com, gitlab.mycompany.com, and sourceforge.net. + github.com, gitlab.mycompany.com, and gitlab.com. The `input.foo` uses the "gitlab" fetcher, which might requires specifying the token type along with the token @@ -70,21 +71,30 @@ Setting warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; - Setting flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", - "Path or URI of the global flake registry."}; + Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", + R"( + Path or URI of the global flake registry. + + When empty, disables the global flake registry. + )", + {}, true, Xp::Flakes}; + Setting useRegistries{this, true, "use-registries", - "Whether to use flake registries to resolve flake references."}; + "Whether to use flake registries to resolve flake references.", + {}, true, Xp::Flakes}; Setting acceptFlakeConfig{this, false, "accept-flake-config", - "Whether to accept nix configuration from a flake without prompting."}; + "Whether to accept nix configuration from a flake without prompting.", + {}, true, Xp::Flakes}; Setting commitLockFileSummary{ this, "", "commit-lockfile-summary", R"( The commit summary to use when committing changed flake lock files. If empty, the summary is generated based on the action performed. - )"}; + )", + {}, true, Xp::Flakes}; }; // FIXME: don't use a global variable. diff -Nru nix-2.8.0/src/libfetchers/git.cc nix-2.16.1+dfsg/src/libfetchers/git.cc --- nix-2.8.0/src/libfetchers/git.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/git.cc 2023-06-06 10:55:23.000000000 +0000 @@ -5,9 +5,13 @@ #include "store-api.hh" #include "url-parts.hh" #include "pathlocks.hh" +#include "util.hh" +#include "git.hh" #include "fetch-settings.hh" +#include +#include #include #include @@ -15,25 +19,237 @@ namespace nix::fetchers { +namespace { + // Explicit initial branch of our bare repo to suppress warnings from new version of git. // The value itself does not matter, since we always fetch a specific revision or branch. // It is set with `-c init.defaultBranch=` instead of `--initial-branch=` to stay compatible with // old version of git, which will ignore unrecognized `-c` options. const std::string gitInitialBranch = "__nix_dummy_branch"; -static std::string readHead(const Path & path) +bool isCacheFileWithinTtl(time_t now, const struct stat & st) +{ + return st.st_mtime + settings.tarballTtl > now; +} + +bool touchCacheFile(const Path & path, time_t touch_time) +{ + struct timeval times[2]; + times[0].tv_sec = touch_time; + times[0].tv_usec = 0; + times[1].tv_sec = touch_time; + times[1].tv_usec = 0; + + return lutimes(path.c_str(), times) == 0; +} + +Path getCachePath(std::string_view key) +{ + return getCacheDir() + "/nix/gitv3/" + + hashString(htSHA256, key).to_string(Base32, false); +} + +// Returns the name of the HEAD branch. +// +// Returns the head branch name as reported by git ls-remote --symref, e.g., if +// ls-remote returns the output below, "main" is returned based on the ref line. +// +// ref: refs/heads/main HEAD +// ... +std::optional readHead(const Path & path) +{ + auto [status, output] = runProgram(RunOptions { + .program = "git", + // FIXME: use 'HEAD' to avoid returning all refs + .args = {"ls-remote", "--symref", path}, + .isInteractive = true, + }); + if (status != 0) return std::nullopt; + + std::string_view line = output; + line = line.substr(0, line.find("\n")); + if (const auto parseResult = git::parseLsRemoteLine(line)) { + switch (parseResult->kind) { + case git::LsRemoteRefLine::Kind::Symbolic: + debug("resolved HEAD ref '%s' for repo '%s'", parseResult->target, path); + break; + case git::LsRemoteRefLine::Kind::Object: + debug("resolved HEAD rev '%s' for repo '%s'", parseResult->target, path); + break; + } + return parseResult->target; + } + return std::nullopt; +} + +// Persist the HEAD ref from the remote repo in the local cached repo. +bool storeCachedHead(const std::string & actualUrl, const std::string & headRef) { - return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" })); + Path cacheDir = getCachePath(actualUrl); + try { + runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef }); + } catch (ExecError &e) { + if (!WIFEXITED(e.status)) throw; + return false; + } + /* No need to touch refs/HEAD, because `git symbolic-ref` updates the mtime. */ + return true; } -static bool isNotDotGitDirectory(const Path & path) +std::optional readHeadCached(const std::string & actualUrl) +{ + // Create a cache path to store the branch of the HEAD ref. Append something + // in front of the URL to prevent collision with the repository itself. + Path cacheDir = getCachePath(actualUrl); + Path headRefFile = cacheDir + "/HEAD"; + + time_t now = time(0); + struct stat st; + std::optional cachedRef; + if (stat(headRefFile.c_str(), &st) == 0) { + cachedRef = readHead(cacheDir); + if (cachedRef != std::nullopt && + *cachedRef != gitInitialBranch && + isCacheFileWithinTtl(now, st)) + { + debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl); + return cachedRef; + } + } + + auto ref = readHead(actualUrl); + if (ref) return ref; + + if (cachedRef) { + // If the cached git ref is expired in fetch() below, and the 'git fetch' + // fails, it falls back to continuing with the most recent version. + // This function must behave the same way, so we return the expired + // cached ref here. + warn("could not get HEAD ref for repository '%s'; using expired cached ref '%s'", actualUrl, *cachedRef); + return *cachedRef; + } + + return std::nullopt; +} + +bool isNotDotGitDirectory(const Path & path) { return baseNameOf(path) != ".git"; } +struct WorkdirInfo +{ + bool clean = false; + bool hasHead = false; +}; + +// Returns whether a git workdir is clean and has commits. +WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) +{ + const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + std::string gitDir(".git"); + + auto env = getEnv(); + // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong + // that way unknown errors can lead to a failure instead of continuing through the wrong code path + env["LC_ALL"] = "C"; + + /* Check whether HEAD points to something that looks like a commit, + since that is the refrence we want to use later on. */ + auto result = runProgram(RunOptions { + .program = "git", + .args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }, + .environment = env, + .mergeStderrToStdout = true + }); + auto exitCode = WEXITSTATUS(result.first); + auto errorMessage = result.second; + + if (errorMessage.find("fatal: not a git repository") != std::string::npos) { + throw Error("'%s' is not a Git repository", workdir); + } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) { + // indicates that the repo does not have any commits + // we want to proceed and will consider it dirty later + } else if (exitCode != 0) { + // any other errors should lead to a failure + throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", workdir, exitCode, errorMessage); + } + + bool clean = false; + bool hasHead = exitCode == 0; + + try { + if (hasHead) { + // Using git diff is preferrable over lower-level operations here, + // because its conceptually simpler and we only need the exit code anyways. + auto gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "diff", "HEAD", "--quiet"}); + if (!submodules) { + // Changes in submodules should only make the tree dirty + // when those submodules will be copied as well. + gitDiffOpts.emplace_back("--ignore-submodules"); + } + gitDiffOpts.emplace_back("--"); + runProgram("git", true, gitDiffOpts); + + clean = true; + } + } catch (ExecError & e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; + } + + return WorkdirInfo { .clean = clean, .hasHead = hasHead }; +} + +std::pair fetchFromWorkdir(ref store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo) +{ + const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + auto gitDir = ".git"; + + if (!fetchSettings.allowDirty) + throw Error("Git tree '%s' is dirty", workdir); + + if (fetchSettings.warnDirty) + warn("Git tree '%s' is dirty", workdir); + + auto gitOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "ls-files", "-z" }); + if (submodules) + gitOpts.emplace_back("--recurse-submodules"); + + auto files = tokenizeString>( + runProgram("git", true, gitOpts), "\0"s); + + Path actualPath(absPath(workdir)); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualPath)); + std::string file(p, actualPath.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); + + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + input.attrs.insert_or_assign( + "lastModified", + workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); + + return {std::move(storePath), input}; +} +} // end namespace + struct GitInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != "git" && url.scheme != "git+http" && @@ -48,10 +264,10 @@ Attrs attrs; attrs.emplace("type", "git"); - for (auto &[name, value] : url.query) { + for (auto & [name, value] : url.query) { if (name == "rev" || name == "ref") attrs.emplace(name, value); - else if (name == "shallow" || name == "submodules") + else if (name == "shallow" || name == "submodules" || name == "allRefs") attrs.emplace(name, Explicit { value == "1" }); else url2.query.emplace(name, value); @@ -62,7 +278,7 @@ return inputFromAttrs(attrs); } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "git") return {}; @@ -85,7 +301,7 @@ return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme != "git") url.scheme = "git+" + url.scheme; @@ -96,7 +312,7 @@ return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { bool maybeDirty = !input.getRef(); bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); @@ -108,7 +324,7 @@ Input applyOverrides( const Input & input, std::optional ref, - std::optional rev) override + std::optional rev) const override { auto res(input); if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); @@ -118,7 +334,7 @@ return res; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { auto [isLocal, actualUrl] = getActualUrl(input); @@ -135,7 +351,7 @@ args.push_back(destDir); - runProgram("git", true, args); + runProgram("git", true, args, {}, true); } std::optional getSourcePath(const Input & input) override @@ -150,13 +366,14 @@ { auto sourcePath = getSourcePath(input); assert(sourcePath); + auto gitDir = ".git"; runProgram("git", true, - { "-C", *sourcePath, "add", "--force", "--intent-to-add", "--", std::string(file) }); + { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) }); if (commitMsg) runProgram("git", true, - { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); + { "-C", *sourcePath, "--git-dir", gitDir, "commit", std::string(file), "-m", *commitMsg }); } std::pair getActualUrl(const Input & input) const @@ -175,6 +392,7 @@ std::pair fetch(ref store, const Input & _input) override { Input input(_input); + auto gitDir = ".git"; std::string name = input.getName(); @@ -223,123 +441,54 @@ auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug - // If this is a local directory and no ref or revision is - // given, then allow the use of an unclean working tree. + /* If this is a local directory and no ref or revision is given, + allow fetching directly from a dirty workdir. */ if (!input.getRef() && !input.getRev() && isLocal) { - bool clean = false; - - auto env = getEnv(); - // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong - // that way unknown errors can lead to a failure instead of continuing through the wrong code path - env["LC_ALL"] = "C"; - - /* Check whether HEAD points to something that looks like a commit, - since that is the refrence we want to use later on. */ - auto result = runProgram(RunOptions { - .program = "git", - .args = { "-C", actualUrl, "--git-dir=.git", "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }, - .environment = env, - .mergeStderrToStdout = true - }); - auto exitCode = WEXITSTATUS(result.first); - auto errorMessage = result.second; - - if (errorMessage.find("fatal: not a git repository") != std::string::npos) { - throw Error("'%s' is not a Git repository", actualUrl); - } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) { - // indicates that the repo does not have any commits - // we want to proceed and will consider it dirty later - } else if (exitCode != 0) { - // any other errors should lead to a failure - throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", actualUrl, exitCode, errorMessage); - } - - bool hasHead = exitCode == 0; - try { - if (hasHead) { - // Using git diff is preferrable over lower-level operations here, - // because its conceptually simpler and we only need the exit code anyways. - auto gitDiffOpts = Strings({ "-C", actualUrl, "diff", "HEAD", "--quiet"}); - if (!submodules) { - // Changes in submodules should only make the tree dirty - // when those submodules will be copied as well. - gitDiffOpts.emplace_back("--ignore-submodules"); - } - gitDiffOpts.emplace_back("--"); - runProgram("git", true, gitDiffOpts); - - clean = true; - } - } catch (ExecError & e) { - if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; - } - - if (!clean) { - - /* This is an unclean working tree. So copy all tracked files. */ - - if (!fetchSettings.allowDirty) - throw Error("Git tree '%s' is dirty", actualUrl); - - if (fetchSettings.warnDirty) - warn("Git tree '%s' is dirty", actualUrl); - - auto gitOpts = Strings({ "-C", actualUrl, "ls-files", "-z" }); - if (submodules) - gitOpts.emplace_back("--recurse-submodules"); - - auto files = tokenizeString>( - runProgram("git", true, gitOpts), "\0"s); - - Path actualPath(absPath(actualUrl)); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, actualPath)); - std::string file(p, actualPath.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); - - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - input.attrs.insert_or_assign( - "lastModified", - hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); - - return {std::move(storePath), input}; + auto workdirInfo = getWorkdirInfo(input, actualUrl); + if (!workdirInfo.clean) { + return fetchFromWorkdir(store, input, actualUrl, workdirInfo); } } - if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master"); - Attrs unlockedAttrs({ {"type", cacheType}, {"name", name}, {"url", actualUrl}, - {"ref", *input.getRef()}, }); Path repoDir; if (isLocal) { + if (!input.getRef()) { + auto head = readHead(actualUrl); + if (!head) { + warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); + head = "master"; + } + input.attrs.insert_or_assign("ref", *head); + unlockedAttrs.insert_or_assign("ref", *head); + } if (!input.getRev()) input.attrs.insert_or_assign("rev", - Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev()); + Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev()); repoDir = actualUrl; - } else { + const bool useHeadRef = !input.getRef(); + if (useHeadRef) { + auto head = readHeadCached(actualUrl); + if (!head) { + warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); + head = "master"; + } + input.attrs.insert_or_assign("ref", *head); + unlockedAttrs.insert_or_assign("ref", *head); + } else { + if (!input.getRev()) { + unlockedAttrs.insert_or_assign("ref", input.getRef().value()); + } + } if (auto res = getCache()->lookup(store, unlockedAttrs)) { auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); @@ -349,8 +498,9 @@ } } - Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); + Path cacheDir = getCachePath(actualUrl); repoDir = cacheDir; + gitDir = "."; createDirs(dirOf(cacheDir)); PathLocks cacheDirLock({cacheDir + ".lock"}); @@ -371,7 +521,7 @@ repo. */ if (input.getRev()) { try { - runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() }); + runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() }); doFetch = false; } catch (ExecError & e) { if (WIFEXITED(e.status)) { @@ -388,7 +538,7 @@ git fetch to update the local ref to the remote ref. */ struct stat st; doFetch = stat(localRefFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + !isCacheFileWithinTtl(now, st); } } @@ -406,19 +556,16 @@ : ref == "HEAD" ? *ref : "refs/heads/" + *ref; - runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); + runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true); } catch (Error & e) { if (!pathExists(localRefFile)) throw; warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); } - struct timeval times[2]; - times[0].tv_sec = now; - times[0].tv_usec = 0; - times[1].tv_sec = now; - times[1].tv_usec = 0; - - utimes(localRefFile.c_str(), times); + if (!touchCacheFile(localRefFile, now)) + warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno)); + if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef())) + warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl); } if (!input.getRev()) @@ -427,10 +574,10 @@ // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } - bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; + bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-parse", "--is-shallow-repository" })) == "true"; if (isShallow && !shallow) - throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); + throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified.", actualUrl); // FIXME: check whether rev is an ancestor of ref. @@ -447,7 +594,7 @@ auto result = runProgram(RunOptions { .program = "git", - .args = { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }, + .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() }, .mergeStderrToStdout = true }); if (WEXITSTATUS(result.first) == 128 @@ -455,9 +602,9 @@ { throw Error( "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " - "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " - ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD - "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", + "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " + ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD + "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", input.getRev()->gitRev(), *input.getRef(), actualUrl @@ -469,15 +616,42 @@ AutoDelete delTmpGitDir(tmpGitDir, true); runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir }); - // TODO: repoDir might lack the ref (it only checks if rev - // exists, see FIXME above) so use a big hammer and fetch - // everything to ensure we get the rev. - runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); + + { + // TODO: repoDir might lack the ref (it only checks if rev + // exists, see FIXME above) so use a big hammer and fetch + // everything to ensure we get the rev. + Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir)); + runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", + "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true); + } runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); - runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); + + /* Ensure that we use the correct origin for fetching + submodules. This matters for submodules with relative + URLs. */ + if (isLocal) { + writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + gitDir + "/config")); + + /* Restore the config.bare setting we may have just + copied erroneously from the user's repo. */ + runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" }); + } else + runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl }); + + /* As an optimisation, copy the modules directory of the + source repo if it exists. */ + auto modulesPath = repoDir + "/" + gitDir + "/modules"; + if (pathExists(modulesPath)) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl)); + runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" }); + } + + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl)); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true); + } filter = isNotDotGitDirectory; } else { @@ -486,7 +660,7 @@ auto source = sinkToSource([&](Sink & sink) { runProgram2({ .program = "git", - .args = { "-C", repoDir, "archive", input.getRev()->gitRev() }, + .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() }, .standardOut = &sink }); }); @@ -496,7 +670,7 @@ auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, @@ -505,7 +679,7 @@ if (!shallow) infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() }))); + std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() }))); if (!_input.getRev()) getCache()->add( diff -Nru nix-2.8.0/src/libfetchers/github.cc nix-2.16.1+dfsg/src/libfetchers/github.cc --- nix-2.8.0/src/libfetchers/github.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/github.cc 2023-06-06 10:55:23.000000000 +0000 @@ -4,7 +4,7 @@ #include "store-api.hh" #include "types.hh" #include "url-parts.hh" - +#include "git.hh" #include "fetchers.hh" #include "fetch-settings.hh" @@ -21,16 +21,16 @@ }; // A github, gitlab, or sourcehut host -const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check +const static std::string hostRegexS = "[a-zA-Z0-9.-]*"; // FIXME: check std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme { - virtual std::string type() = 0; + virtual std::string type() const = 0; virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != type()) return {}; @@ -100,7 +100,7 @@ return input; } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != type()) return {}; @@ -116,7 +116,7 @@ return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto owner = getStrAttr(input.attrs, "owner"); auto repo = getStrAttr(input.attrs, "repo"); @@ -132,7 +132,7 @@ }; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); } @@ -140,7 +140,7 @@ Input applyOverrides( const Input & _input, std::optional ref, - std::optional rev) override + std::optional rev) const override { auto input(_input); if (rev && ref) @@ -227,7 +227,7 @@ struct GitHubInputScheme : GitArchiveInputScheme { - std::string type() override { return "github"; } + std::string type() const override { return "github"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -240,11 +240,29 @@ return std::pair("Authorization", fmt("token %s", token)); } + std::string getHost(const Input & input) const + { + return maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + } + + std::string getOwner(const Input & input) const + { + return getStrAttr(input.attrs, "owner"); + } + + std::string getRepo(const Input & input) const + { + return getStrAttr(input.attrs, "repo"); + } + Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); - auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check - host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + auto host = getHost(input); + auto url = fmt( + host == "github.com" + ? "https://api.%s/repos/%s/%s/commits/%s" + : "https://%s/api/v3/repos/%s/%s/commits/%s", + host, getOwner(input), getRepo(input), *input.getRef()); Headers headers = makeHeadersWithAuthTokens(host); @@ -259,22 +277,30 @@ DownloadUrl getDownloadUrl(const Input & input) const override { - // FIXME: use regular /archive URLs instead? api.github.com - // might have stricter rate limits. - auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); - auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances - host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + auto host = getHost(input); Headers headers = makeHeadersWithAuthTokens(host); + + // If we have no auth headers then we default to the public archive + // urls so we do not run into rate limits. + const auto urlFmt = + host != "github.com" + ? "https://%s/api/v3/repos/%s/%s/tarball/%s" + : headers.empty() + ? "https://%s/%s/%s/archive/%s.tar.gz" + : "https://api.%s/repos/%s/%s/tarball/%s"; + + const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), + input.getRev()->to_string(Base16, false)); + return DownloadUrl { url, headers }; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { - auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + auto host = getHost(input); Input::fromURL(fmt("git+https://%s/%s/%s.git", - host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + host, getOwner(input), getRepo(input))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); } @@ -282,7 +308,7 @@ struct GitLabInputScheme : GitArchiveInputScheme { - std::string type() override { return "gitlab"; } + std::string type() const override { return "gitlab"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -337,7 +363,7 @@ return DownloadUrl { url, headers }; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // FIXME: get username somewhere @@ -350,7 +376,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme { - std::string type() override { return "sourcehut"; } + std::string type() const override { return "sourcehut"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -375,7 +401,7 @@ Headers headers = makeHeadersWithAuthTokens(host); - std::string ref_uri; + std::string refUri; if (ref == "HEAD") { auto file = store->toRealPath( downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath); @@ -383,35 +409,32 @@ std::string line; getline(is, line); - auto ref_index = line.find("ref: "); - if (ref_index == std::string::npos) { + auto remoteLine = git::parseLsRemoteLine(line); + if (!remoteLine) { throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref); } - - ref_uri = line.substr(ref_index+5, line.length()-1); - } else - ref_uri = fmt("refs/(heads|tags)/%s", ref); + refUri = remoteLine->target; + } else { + refUri = fmt("refs/(heads|tags)/%s", ref); + } + std::regex refRegex(refUri); auto file = store->toRealPath( downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath); std::ifstream is(file); std::string line; - std::string id; - while(getline(is, line)) { - // Append $ to avoid partial name matches - std::regex pattern(fmt("%s$", ref_uri)); - - if (std::regex_search(line, pattern)) { - id = line.substr(0, line.find('\t')); - break; - } + std::optional id; + while(!id && getline(is, line)) { + auto parsedLine = git::parseLsRemoteLine(line); + if (parsedLine && parsedLine->reference && std::regex_match(*parsedLine->reference, refRegex)) + id = parsedLine->target; } - if(id.empty()) + if(!id) throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref); - auto rev = Hash::parseAny(id, htSHA1); + auto rev = Hash::parseAny(*id, htSHA1); debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev()); return rev; } @@ -427,7 +450,7 @@ return DownloadUrl { url, headers }; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); Input::fromURL(fmt("git+https://%s/%s/%s", diff -Nru nix-2.8.0/src/libfetchers/indirect.cc nix-2.16.1+dfsg/src/libfetchers/indirect.cc --- nix-2.8.0/src/libfetchers/indirect.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/indirect.cc 2023-06-06 10:55:23.000000000 +0000 @@ -7,7 +7,7 @@ struct IndirectInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != "flake") return {}; @@ -50,7 +50,7 @@ return input; } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; @@ -68,7 +68,7 @@ return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { ParsedURL url; url.scheme = "flake"; @@ -78,7 +78,7 @@ return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return false; } @@ -86,7 +86,7 @@ Input applyOverrides( const Input & _input, std::optional ref, - std::optional rev) override + std::optional rev) const override { auto input(_input); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); diff -Nru nix-2.8.0/src/libfetchers/input-accessor.cc nix-2.16.1+dfsg/src/libfetchers/input-accessor.cc --- nix-2.8.0/src/libfetchers/input-accessor.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/input-accessor.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,106 @@ +#include "input-accessor.hh" +#include "store-api.hh" + +namespace nix { + +std::ostream & operator << (std::ostream & str, const SourcePath & path) +{ + str << path.to_string(); + return str; +} + +std::string_view SourcePath::baseName() const +{ + return path.baseName().value_or("source"); +} + +SourcePath SourcePath::parent() const +{ + auto p = path.parent(); + assert(p); + return std::move(*p); +} + +InputAccessor::Stat SourcePath::lstat() const +{ + auto st = nix::lstat(path.abs()); + return InputAccessor::Stat { + .type = + S_ISREG(st.st_mode) ? InputAccessor::tRegular : + S_ISDIR(st.st_mode) ? InputAccessor::tDirectory : + S_ISLNK(st.st_mode) ? InputAccessor::tSymlink : + InputAccessor::tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; +} + +std::optional SourcePath::maybeLstat() const +{ + // FIXME: merge these into one operation. + if (!pathExists()) + return {}; + return lstat(); +} + +InputAccessor::DirEntries SourcePath::readDirectory() const +{ + InputAccessor::DirEntries res; + for (auto & entry : nix::readDirectory(path.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = InputAccessor::Type::tRegular; break; + case DT_LNK: type = InputAccessor::Type::tSymlink; break; + case DT_DIR: type = InputAccessor::Type::tDirectory; break; + } + res.emplace(entry.name, type); + } + return res; +} + +StorePath SourcePath::fetchToStore( + ref store, + std::string_view name, + PathFilter * filter, + RepairFlag repair) const +{ + return + settings.readOnlyMode + ? store->computeStorePathForPath(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter).first + : store->addToStore(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter, repair); +} + +SourcePath SourcePath::resolveSymlinks() const +{ + SourcePath res(CanonPath::root); + + int linksAllowed = 1024; + + std::list todo; + for (auto & c : path) + todo.push_back(std::string(c)); + + while (!todo.empty()) { + auto c = *todo.begin(); + todo.pop_front(); + if (c == "" || c == ".") + ; + else if (c == "..") + res.path.pop(); + else { + res.path.push(c); + if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) { + if (!linksAllowed--) + throw Error("infinite symlink recursion in path '%s'", path); + auto target = res.readLink(); + res.path.pop(); + if (hasPrefix(target, "/")) + res.path = CanonPath::root; + todo.splice(todo.begin(), tokenizeString>(target, "/")); + } + } + } + + return res; +} + +} diff -Nru nix-2.8.0/src/libfetchers/input-accessor.hh nix-2.16.1+dfsg/src/libfetchers/input-accessor.hh --- nix-2.8.0/src/libfetchers/input-accessor.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/input-accessor.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,167 @@ +#pragma once + +#include "ref.hh" +#include "types.hh" +#include "archive.hh" +#include "canon-path.hh" +#include "repair-flag.hh" + +namespace nix { + +class StorePath; +class Store; + +struct InputAccessor +{ + enum Type { + tRegular, tSymlink, tDirectory, + /** + Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. + + Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`. + + Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. + */ + tMisc + }; + + struct Stat + { + Type type = tMisc; + //uint64_t fileSize = 0; // regular files only + bool isExecutable = false; // regular files only + }; + + typedef std::optional DirEntry; + + typedef std::map DirEntries; +}; + +/** + * An abstraction for accessing source files during + * evaluation. Currently, it's just a wrapper around `CanonPath` that + * accesses files in the regular filesystem, but in the future it will + * support fetching files in other ways. + */ +struct SourcePath +{ + CanonPath path; + + SourcePath(CanonPath path) + : path(std::move(path)) + { } + + std::string_view baseName() const; + + /** + * Construct the parent of this `SourcePath`. Aborts if `this` + * denotes the root. + */ + SourcePath parent() const; + + /** + * If this `SourcePath` denotes a regular file (not a symlink), + * return its contents; otherwise throw an error. + */ + std::string readFile() const + { return nix::readFile(path.abs()); } + + /** + * Return whether this `SourcePath` denotes a file (of any type) + * that exists + */ + bool pathExists() const + { return nix::pathExists(path.abs()); } + + /** + * Return stats about this `SourcePath`, or throw an exception if + * it doesn't exist. + */ + InputAccessor::Stat lstat() const; + + /** + * Return stats about this `SourcePath`, or std::nullopt if it + * doesn't exist. + */ + std::optional maybeLstat() const; + + /** + * If this `SourcePath` denotes a directory (not a symlink), + * return its directory entries; otherwise throw an error. + */ + InputAccessor::DirEntries readDirectory() const; + + /** + * If this `SourcePath` denotes a symlink, return its target; + * otherwise throw an error. + */ + std::string readLink() const + { return nix::readLink(path.abs()); } + + /** + * Dump this `SourcePath` to `sink` as a NAR archive. + */ + void dumpPath( + Sink & sink, + PathFilter & filter = defaultPathFilter) const + { return nix::dumpPath(path.abs(), sink, filter); } + + /** + * Copy this `SourcePath` to the Nix store. + */ + StorePath fetchToStore( + ref store, + std::string_view name = "source", + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair) const; + + /** + * Return the location of this path in the "real" filesystem, if + * it has a physical location. + */ + std::optional getPhysicalPath() const + { return path; } + + std::string to_string() const + { return path.abs(); } + + /** + * Append a `CanonPath` to this path. + */ + SourcePath operator + (const CanonPath & x) const + { return {path + x}; } + + /** + * Append a single component `c` to this path. `c` must not + * contain a slash. A slash is implicitly added between this path + * and `c`. + */ + SourcePath operator + (std::string_view c) const + { return {path + c}; } + + bool operator == (const SourcePath & x) const + { + return path == x.path; + } + + bool operator != (const SourcePath & x) const + { + return path != x.path; + } + + bool operator < (const SourcePath & x) const + { + return path < x.path; + } + + /** + * Resolve any symlinks in this `SourcePath` (including its + * parents). The result is a `SourcePath` in which no element is a + * symlink. + */ + SourcePath resolveSymlinks() const; +}; + +std::ostream & operator << (std::ostream & str, const SourcePath & path); + +} diff -Nru nix-2.8.0/src/libfetchers/mercurial.cc nix-2.16.1+dfsg/src/libfetchers/mercurial.cc --- nix-2.8.0/src/libfetchers/mercurial.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/mercurial.cc 2023-06-06 10:55:23.000000000 +0000 @@ -36,14 +36,14 @@ auto res = runProgram(std::move(opts)); if (!statusOk(res.first)) - throw ExecError(res.first, fmt("hg %1%", statusToString(res.first))); + throw ExecError(res.first, "hg %1%", statusToString(res.first)); return res.second; } struct MercurialInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && @@ -69,7 +69,7 @@ return inputFromAttrs(attrs); } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "hg") return {}; @@ -89,7 +89,7 @@ return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); url.scheme = "hg+" + url.scheme; @@ -98,7 +98,7 @@ return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { // FIXME: ugly, need to distinguish between dirty and clean // default trees. @@ -108,7 +108,7 @@ Input applyOverrides( const Input & input, std::optional ref, - std::optional rev) override + std::optional rev) const override { auto res(input); if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); @@ -273,7 +273,7 @@ runHg({ "recover", "-R", cacheDir }); runHg({ "pull", "-R", cacheDir, "--", actualUrl }); } else { - throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); + throw ExecError(e.status, "'hg pull' %s", statusToString(e.status)); } } } else { diff -Nru nix-2.8.0/src/libfetchers/path.cc nix-2.16.1+dfsg/src/libfetchers/path.cc --- nix-2.8.0/src/libfetchers/path.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/path.cc 2023-06-06 10:55:23.000000000 +0000 @@ -6,7 +6,7 @@ struct PathInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != "path") return {}; @@ -32,7 +32,7 @@ return input; } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "path") return {}; @@ -54,7 +54,7 @@ return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto query = attrsToQuery(input.attrs); query.erase("path"); @@ -66,7 +66,7 @@ }; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return true; } diff -Nru nix-2.8.0/src/libfetchers/registry.cc nix-2.16.1+dfsg/src/libfetchers/registry.cc --- nix-2.8.0/src/libfetchers/registry.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/registry.cc 2023-06-06 10:55:23.000000000 +0000 @@ -153,6 +153,9 @@ { static auto reg = [&]() { auto path = fetchSettings.flakeRegistry.get(); + if (path == "") { + return std::make_shared(Registry::Global); // empty registry + } if (!hasPrefix(path, "/")) { auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; diff -Nru nix-2.8.0/src/libfetchers/registry.hh nix-2.16.1+dfsg/src/libfetchers/registry.hh --- nix-2.8.0/src/libfetchers/registry.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/registry.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "fetchers.hh" diff -Nru nix-2.8.0/src/libfetchers/tarball.cc nix-2.16.1+dfsg/src/libfetchers/tarball.cc --- nix-2.8.0/src/libfetchers/tarball.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libfetchers/tarball.cc 2023-06-06 10:55:23.000000000 +0000 @@ -6,6 +6,7 @@ #include "archive.hh" #include "tarfile.hh" #include "types.hh" +#include "split.hh" namespace nix::fetchers { @@ -70,15 +71,19 @@ dumpString(res.data, sink); auto hash = hashString(htSHA256, res.data); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), + *store, + name, + FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, + .references = {}, + }, hashString(htSHA256, sink.s), }; info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = hash, - }; - auto source = StringSource(sink.s); + auto source = StringSource { sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); } @@ -168,63 +173,108 @@ }; } -struct TarballInputScheme : InputScheme +// An input scheme corresponding to a curl-downloadable resource. +struct CurlInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + virtual const std::string inputType() const = 0; + const std::set transportUrlSchemes = {"file", "http", "https"}; + + const bool hasTarballExtension(std::string_view path) const { - if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {}; + return hasSuffix(path, ".zip") || hasSuffix(path, ".tar") + || hasSuffix(path, ".tgz") || hasSuffix(path, ".tar.gz") + || hasSuffix(path, ".tar.xz") || hasSuffix(path, ".tar.bz2") + || hasSuffix(path, ".tar.zst"); + } - if (!hasSuffix(url.path, ".zip") - && !hasSuffix(url.path, ".tar") - && !hasSuffix(url.path, ".tgz") - && !hasSuffix(url.path, ".tar.gz") - && !hasSuffix(url.path, ".tar.xz") - && !hasSuffix(url.path, ".tar.bz2") - && !hasSuffix(url.path, ".tar.zst")) - return {}; + virtual bool isValidURL(const ParsedURL & url) const = 0; + + std::optional inputFromURL(const ParsedURL & url) const override + { + if (!isValidURL(url)) + return std::nullopt; Input input; - input.attrs.insert_or_assign("type", "tarball"); - input.attrs.insert_or_assign("url", url.to_string()); + + auto urlWithoutApplicationScheme = url; + urlWithoutApplicationScheme.scheme = parseUrlScheme(url.scheme).transport; + + input.attrs.insert_or_assign("type", inputType()); + input.attrs.insert_or_assign("url", urlWithoutApplicationScheme.to_string()); auto narHash = url.query.find("narHash"); if (narHash != url.query.end()) input.attrs.insert_or_assign("narHash", narHash->second); return input; } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; + auto type = maybeGetStrAttr(attrs, "type"); + if (type != inputType()) return {}; + std::set allowedNames = {"type", "url", "narHash", "name", "unpack"}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name") - throw Error("unsupported tarball input attribute '%s'", name); + if (!allowedNames.count(name)) + throw Error("unsupported %s input attribute '%s'", *type, name); Input input; input.attrs = attrs; + //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash"); return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); - // NAR hashes are preferred over file hashes since tar/zip files - // don't have a canonical representation. + // NAR hashes are preferred over file hashes since tar/zip + // files don't have a canonical representation. if (auto narHash = input.getNarHash()) url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); - /* - else if (auto hash = maybeGetStrAttr(input.attrs, "hash")) - url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true)); - */ return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return true; } +}; + +struct FileInputScheme : CurlInputScheme +{ + const std::string inputType() const override { return "file"; } + + bool isValidURL(const ParsedURL & url) const override + { + auto parsedUrlScheme = parseUrlScheme(url.scheme); + return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) + && (parsedUrlScheme.application + ? parsedUrlScheme.application.value() == inputType() + : !hasTarballExtension(url.path)); + } + + std::pair fetch(ref store, const Input & input) override + { + auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false); + return {std::move(file.storePath), input}; + } +}; + +struct TarballInputScheme : CurlInputScheme +{ + const std::string inputType() const override { return "tarball"; } + + bool isValidURL(const ParsedURL & url) const override + { + auto parsedUrlScheme = parseUrlScheme(url.scheme); + + return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) + && (parsedUrlScheme.application + ? parsedUrlScheme.application.value() == inputType() + : hasTarballExtension(url.path)); + } + std::pair fetch(ref store, const Input & input) override { auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first; @@ -233,5 +283,6 @@ }; static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rFileInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff -Nru nix-2.8.0/src/libmain/common-args.cc nix-2.16.1+dfsg/src/libmain/common-args.cc --- nix-2.8.0/src/libmain/common-args.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libmain/common-args.cc 2023-06-06 10:55:23.000000000 +0000 @@ -32,6 +32,7 @@ addFlag({ .longName = "option", .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).", + .category = miscCategory, .labels = {"name", "value"}, .handler = {[](std::string name, std::string value) { try { diff -Nru nix-2.8.0/src/libmain/common-args.hh nix-2.16.1+dfsg/src/libmain/common-args.hh --- nix-2.8.0/src/libmain/common-args.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libmain/common-args.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,11 +1,14 @@ #pragma once +///@file #include "args.hh" +#include "repair-flag.hh" namespace nix { //static constexpr auto commonArgsCategory = "Miscellaneous common options"; static constexpr auto loggingCategory = "Logging-related options"; +static constexpr auto miscCategory = "Miscellaneous global options"; class MixCommonArgs : public virtual Args { @@ -46,5 +49,22 @@ }); } }; + +struct MixRepair : virtual Args +{ + RepairFlag repair = NoRepair; + + MixRepair() + { + addFlag({ + .longName = "repair", + .description = + "During evaluation, rewrite missing or corrupted files in the Nix store. " + "During building, rebuild missing or corrupted store paths.", + .category = miscCategory, + .handler = {&repair, Repair}, + }); + } +}; } diff -Nru nix-2.8.0/src/libmain/loggers.cc nix-2.16.1+dfsg/src/libmain/loggers.cc --- nix-2.8.0/src/libmain/loggers.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libmain/loggers.cc 2023-06-06 10:55:23.000000000 +0000 @@ -30,8 +30,11 @@ return makeJSONLogger(*makeSimpleLogger(true)); case LogFormat::bar: return makeProgressBar(); - case LogFormat::barWithLogs: - return makeProgressBar(true); + case LogFormat::barWithLogs: { + auto logger = makeProgressBar(); + logger->setPrintBuildLogs(true); + return logger; + } default: abort(); } diff -Nru nix-2.8.0/src/libmain/loggers.hh nix-2.16.1+dfsg/src/libmain/loggers.hh --- nix-2.8.0/src/libmain/loggers.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libmain/loggers.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff -Nru nix-2.8.0/src/libmain/nix-main.pc.in nix-2.16.1+dfsg/src/libmain/nix-main.pc.in --- nix-2.8.0/src/libmain/nix-main.pc.in 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libmain/nix-main.pc.in 2023-06-06 10:55:23.000000000 +0000 @@ -6,4 +6,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff -Nru nix-2.8.0/src/libmain/progress-bar.cc nix-2.16.1+dfsg/src/libmain/progress-bar.cc --- nix-2.8.0/src/libmain/progress-bar.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libmain/progress-bar.cc 2023-06-06 10:55:23.000000000 +0000 @@ -8,6 +8,7 @@ #include #include #include +#include namespace nix { @@ -48,6 +49,7 @@ bool visible = true; ActivityId parent; std::optional name; + std::chrono::time_point startTime; }; struct ActivitiesByType @@ -70,6 +72,7 @@ uint64_t corruptedPaths = 0, untrustedPaths = 0; bool active = true; + bool paused = false; bool haveUpdate = true; }; @@ -79,22 +82,22 @@ std::condition_variable quitCV, updateCV; - bool printBuildLogs; + bool printBuildLogs = false; bool isTTY; public: - ProgressBar(bool printBuildLogs, bool isTTY) - : printBuildLogs(printBuildLogs) - , isTTY(isTTY) + ProgressBar(bool isTTY) + : isTTY(isTTY) { state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); + auto nextWakeup = std::chrono::milliseconds::max(); while (state->active) { if (!state->haveUpdate) - state.wait(updateCV); - draw(*state); + state.wait_for(updateCV, nextWakeup); + nextWakeup = draw(*state); state.wait_for(quitCV, std::chrono::milliseconds(50)); } }); @@ -118,18 +121,31 @@ updateThread.join(); } - bool isVerbose() override { + void pause() override { + state_.lock()->paused = true; + writeToStderr("\r\e[K"); + } + + void resume() override { + state_.lock()->paused = false; + writeToStderr("\r\e[K"); + state_.lock()->haveUpdate = true; + updateCV.notify_one(); + } + + bool isVerbose() override + { return printBuildLogs; } - void log(Verbosity lvl, const FormatOrString & fs) override + void log(Verbosity lvl, std::string_view s) override { if (lvl > verbosity) return; auto state(state_.lock()); - log(*state, lvl, fs.s); + log(*state, lvl, s); } - void logEI(const ErrorInfo &ei) override + void logEI(const ErrorInfo & ei) override { auto state(state_.lock()); @@ -139,7 +155,7 @@ log(*state, ei.level, oss.str()); } - void log(State & state, Verbosity lvl, const std::string & s) + void log(State & state, Verbosity lvl, std::string_view s) { if (state.active) { writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); @@ -159,11 +175,13 @@ if (lvl <= verbosity && !s.empty() && type != actBuildWaiting) log(*state, lvl, s + "..."); - state->activities.emplace_back(ActInfo()); + state->activities.emplace_back(ActInfo { + .s = s, + .type = type, + .parent = parent, + .startTime = std::chrono::steady_clock::now() + }); auto i = std::prev(state->activities.end()); - i->s = s; - i->type = type; - i->parent = parent; state->its.emplace(act, i); state->activitiesByType[type].its.emplace(act, i); @@ -175,10 +193,12 @@ auto machineName = getS(fields, 1); if (machineName != "") i->s += fmt(" on " ANSI_BOLD "%s" ANSI_NORMAL, machineName); - auto curRound = getI(fields, 2); - auto nrRounds = getI(fields, 3); - if (nrRounds != 1) - i->s += fmt(" (round %d/%d)", curRound, nrRounds); + + // Used to be curRound and nrRounds, but the + // implementation was broken for a long time. + if (getI(fields, 2) != 1 || getI(fields, 3) != 1) { + throw Error("log message indicated repeating builds, but this is not currently implemented"); + } i->name = DrvName(name).name; } @@ -327,10 +347,12 @@ updateCV.notify_one(); } - void draw(State & state) + std::chrono::milliseconds draw(State & state) { + auto nextWakeup = std::chrono::milliseconds::max(); + state.haveUpdate = false; - if (!state.active) return; + if (state.paused || !state.active) return nextWakeup; std::string line; @@ -341,12 +363,25 @@ line += "]"; } + auto now = std::chrono::steady_clock::now(); + if (!state.activities.empty()) { if (!status.empty()) line += " "; auto i = state.activities.rbegin(); - while (i != state.activities.rend() && (!i->visible || (i->s.empty() && i->lastLine.empty()))) + while (i != state.activities.rend()) { + if (i->visible && (!i->s.empty() || !i->lastLine.empty())) { + /* Don't show activities until some time has + passed, to avoid displaying very short + activities. */ + auto delay = std::chrono::milliseconds(10); + if (i->startTime + delay < now) + break; + else + nextWakeup = std::min(nextWakeup, std::chrono::duration_cast(delay - (now - i->startTime))); + } ++i; + } if (i != state.activities.rend()) { line += i->s; @@ -366,6 +401,8 @@ if (width <= 0) width = std::numeric_limits::max(); writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K"); + + return nextWakeup; } std::string getStatus(State & state) @@ -480,19 +517,21 @@ draw(*state); return s[0]; } + + void setPrintBuildLogs(bool printBuildLogs) override + { + this->printBuildLogs = printBuildLogs; + } }; -Logger * makeProgressBar(bool printBuildLogs) +Logger * makeProgressBar() { - return new ProgressBar( - printBuildLogs, - shouldANSI() - ); + return new ProgressBar(shouldANSI()); } -void startProgressBar(bool printBuildLogs) +void startProgressBar() { - logger = makeProgressBar(printBuildLogs); + logger = makeProgressBar(); } void stopProgressBar() diff -Nru nix-2.8.0/src/libmain/progress-bar.hh nix-2.16.1+dfsg/src/libmain/progress-bar.hh --- nix-2.8.0/src/libmain/progress-bar.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libmain/progress-bar.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,12 +1,13 @@ #pragma once +///@file #include "logging.hh" namespace nix { -Logger * makeProgressBar(bool printBuildLogs = false); +Logger * makeProgressBar(); -void startProgressBar(bool printBuildLogs = false); +void startProgressBar(); void stopProgressBar(); diff -Nru nix-2.8.0/src/libmain/shared.cc nix-2.16.1+dfsg/src/libmain/shared.cc --- nix-2.8.0/src/libmain/shared.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libmain/shared.cc 2023-06-06 10:55:23.000000000 +0000 @@ -4,12 +4,12 @@ #include "gc-store.hh" #include "util.hh" #include "loggers.hh" +#include "progress-bar.hh" #include #include #include #include -#include #include #include @@ -19,19 +19,13 @@ #ifdef __linux__ #include #endif -#ifdef __GLIBC__ -#include -#include -#include -#endif #include -#include - namespace nix { +char * * savedArgv; static bool gcWarning = true; @@ -60,37 +54,47 @@ { if (!willBuild.empty()) { if (willBuild.size() == 1) - printMsg(lvl, fmt("this derivation will be built:")); + printMsg(lvl, "this derivation will be built:"); else - printMsg(lvl, fmt("these %d derivations will be built:", willBuild.size())); + printMsg(lvl, "these %d derivations will be built:", willBuild.size()); auto sorted = store->topoSortPaths(willBuild); reverse(sorted.begin(), sorted.end()); for (auto & i : sorted) - printMsg(lvl, fmt(" %s", store->printStorePath(i))); + printMsg(lvl, " %s", store->printStorePath(i)); } if (!willSubstitute.empty()) { const float downloadSizeMiB = downloadSize / (1024.f * 1024.f); const float narSizeMiB = narSize / (1024.f * 1024.f); if (willSubstitute.size() == 1) { - printMsg(lvl, fmt("this path will be fetched (%.2f MiB download, %.2f MiB unpacked):", + printMsg(lvl, "this path will be fetched (%.2f MiB download, %.2f MiB unpacked):", downloadSizeMiB, - narSizeMiB)); + narSizeMiB); } else { - printMsg(lvl, fmt("these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", + printMsg(lvl, "these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", willSubstitute.size(), downloadSizeMiB, - narSizeMiB)); + narSizeMiB); } - for (auto & i : willSubstitute) - printMsg(lvl, fmt(" %s", store->printStorePath(i))); + std::vector willSubstituteSorted = {}; + std::for_each(willSubstitute.begin(), willSubstitute.end(), + [&](const StorePath &p) { willSubstituteSorted.push_back(&p); }); + std::sort(willSubstituteSorted.begin(), willSubstituteSorted.end(), + [](const StorePath *lhs, const StorePath *rhs) { + if (lhs->name() == rhs->name()) + return lhs->to_string() < rhs->to_string(); + else + return lhs->name() < rhs->name(); + }); + for (auto p : willSubstituteSorted) + printMsg(lvl, " %s", store->printStorePath(*p)); } if (!unknown.empty()) { - printMsg(lvl, fmt("don't know how to build these paths%s:", - (settings.readOnlyMode ? " (may be caused by read-only store access)" : ""))); + printMsg(lvl, "don't know how to build these paths%s:", + (settings.readOnlyMode ? " (may be caused by read-only store access)" : "")); for (auto & i : unknown) - printMsg(lvl, fmt(" %s", store->printStorePath(i))); + printMsg(lvl, " %s", store->printStorePath(i)); } } @@ -103,57 +107,6 @@ return *i; } - -#if OPENSSL_VERSION_NUMBER < 0x10101000L -/* OpenSSL is not thread-safe by default - it will randomly crash - unless the user supplies a mutex locking function. So let's do - that. */ -static std::vector opensslLocks; - -static void opensslLockCallback(int mode, int type, const char * file, int line) -{ - if (mode & CRYPTO_LOCK) - opensslLocks[type].lock(); - else - opensslLocks[type].unlock(); -} -#endif - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { -#ifdef __GLIBC__ - /* On linux, glibc will run every lookup through the nss layer. - * That means every lookup goes, by default, through nscd, which acts as a local - * cache. - * Because we run builds in a sandbox, we also remove access to nscd otherwise - * lookups would leak into the sandbox. - * - * But now we have a new problem, we need to make sure the nss_dns backend that - * does the dns lookups when nscd is not available is loaded or available. - * - * We can't make it available without leaking nix's environment, so instead we'll - * load the backend, and configure nss so it does not try to run dns lookups - * through nscd. - * - * This is technically only used for builtins:fetch* functions so we only care - * about dns. - * - * All other platforms are unaffected. - */ - if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) - warn("unable to load nss_dns backend"); - // FIXME: get hosts entry from nsswitch.conf. - __nss_configure_lookup("hosts", "files dns"); -#endif - }); -} - static void sigHandler(int signo) { } @@ -165,24 +118,16 @@ std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif -#if OPENSSL_VERSION_NUMBER < 0x10101000L - /* Initialise OpenSSL locking. */ - opensslLocks = std::vector(CRYPTO_num_locks()); - CRYPTO_set_locking_callback(opensslLockCallback); -#endif - - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - - loadConfFile(); + initLibStore(); startSignalHandlerThread(); /* Reset SIGCHLD to its default. */ struct sigaction act; sigemptyset(&act.sa_mask); - act.sa_handler = SIG_DFL; act.sa_flags = 0; + + act.sa_handler = SIG_DFL; if (sigaction(SIGCHLD, &act, 0)) throw SysError("resetting SIGCHLD"); @@ -194,12 +139,26 @@ /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. * Instead, add a dummy sigaction handler, and signalHandlerThread * can handle the rest. */ - struct sigaction sa; - sa.sa_handler = sigHandler; - if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH"); + act.sa_handler = sigHandler; + if (sigaction(SIGWINCH, &act, 0)) throw SysError("handling SIGWINCH"); + + /* Disable SA_RESTART for interrupts, so that system calls on this thread + * error with EINTR like they do on Linux. + * Most signals on BSD systems default to SA_RESTART on, but Nix + * expects EINTR from syscalls to properly exit. */ + act.sa_handler = SIG_DFL; + if (sigaction(SIGINT, &act, 0)) throw SysError("handling SIGINT"); + if (sigaction(SIGTERM, &act, 0)) throw SysError("handling SIGTERM"); + if (sigaction(SIGHUP, &act, 0)) throw SysError("handling SIGHUP"); + if (sigaction(SIGPIPE, &act, 0)) throw SysError("handling SIGPIPE"); + if (sigaction(SIGQUIT, &act, 0)) throw SysError("handling SIGQUIT"); + if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP"); #endif - /* Register a SIGSEGV handler to detect stack overflows. */ + /* Register a SIGSEGV handler to detect stack overflows. + Why not initLibExpr()? initGC() is essentially that, but + detectStackOverflow is not an instance of the init function concept, as + it may have to be invoked more than once per process. */ detectStackOverflow(); /* There is no privacy in the Nix system ;-) At least not for @@ -212,15 +171,6 @@ gettimeofday(&tv, 0); srandom(tv.tv_usec); - /* On macOS, don't use the per-session TMPDIR (as set e.g. by - sshd). This breaks build users because they don't have access - to the TMPDIR, in particular in ‘nix-store --serve’. */ -#if __APPLE__ - if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) - unsetenv("TMPDIR"); -#endif - - preloadNSS(); } @@ -332,7 +282,7 @@ void printVersion(const std::string & programName) { - std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl; + std::cout << fmt("%1% (Nix) %2%", programName, nixVersion) << std::endl; if (verbosity > lvlInfo) { Strings cfg; #if HAVE_BOEHMGC @@ -348,6 +298,7 @@ << "\n"; std::cout << "Store directory: " << settings.nixStore << "\n"; std::cout << "State directory: " << settings.nixStateDir << "\n"; + std::cout << "Data directory: " << settings.nixDataDir << "\n"; } throw Exit(); } @@ -388,8 +339,6 @@ return 1; } catch (BaseError & e) { logError(e.info()); - if (e.hasTrace() && !loggerSettings.showTrace.get()) - printError("(use '--show-trace' to show detailed location information)"); return e.status; } catch (std::bad_alloc & e) { printError(error + "out of memory"); @@ -410,6 +359,8 @@ if (!pager) pager = getenv("PAGER"); if (pager && ((std::string) pager == "" || (std::string) pager == "cat")) return; + stopProgressBar(); + Pipe toPager; toPager.create(); diff -Nru nix-2.8.0/src/libmain/shared.hh nix-2.16.1+dfsg/src/libmain/shared.hh --- nix-2.8.0/src/libmain/shared.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libmain/shared.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "args.hh" @@ -24,7 +25,9 @@ int handleExceptions(const std::string & programName, std::function fun); -/* Don't forget to call initPlugins() after settings are initialized! */ +/** + * Don't forget to call initPlugins() after settings are initialized! + */ void initNix(); void parseCmdLine(int argc, char * * argv, @@ -35,11 +38,12 @@ void printVersion(const std::string & programName); -/* Ugh. No better place to put this. */ +/** + * Ugh. No better place to put this. + */ void printGCWarning(); class Store; -struct StorePathWithOutputs; void printMissing( ref store, @@ -75,11 +79,15 @@ }; -/* Show the manual page for the specified program. */ +/** + * Show the manual page for the specified program. + */ void showManPage(const std::string & name); -/* The constructor of this class starts a pager if stdout is a - terminal and $PAGER is set. Stdout is redirected to the pager. */ +/** + * The constructor of this class starts a pager if stdout is a + * terminal and $PAGER is set. Stdout is redirected to the pager. + */ class RunPager { public: @@ -110,8 +118,34 @@ }; -/* Install a SIGSEGV handler to detect stack overflows. */ +/** + * Install a SIGSEGV handler to detect stack overflows. + */ void detectStackOverflow(); +/** + * Pluggable behavior to run in case of a stack overflow. + * + * Default value: defaultStackOverflowHandler. + * + * This is called by the handler installed by detectStackOverflow(). + * + * This gives Nix library consumers a limit opportunity to report the error + * condition. The handler should exit the process. + * See defaultStackOverflowHandler() for a reference implementation. + * + * NOTE: Use with diligence, because this runs in the signal handler, with very + * limited stack space and a potentially a corrupted heap, all while the failed + * thread is blocked indefinitely. All functions called must be reentrant. + */ +extern std::function stackOverflowHandler; + +/** + * The default, robust implementation of stackOverflowHandler. + * + * Prints an error message directly to stderr using a syscall instead of the + * logger. Exits the process immediately after. + */ +void defaultStackOverflowHandler(siginfo_t * info, void * ctx); } diff -Nru nix-2.8.0/src/libmain/stack.cc nix-2.16.1+dfsg/src/libmain/stack.cc --- nix-2.8.0/src/libmain/stack.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libmain/stack.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #include "error.hh" +#include "shared.hh" #include #include @@ -29,9 +30,7 @@ ptrdiff_t diff = (char *) info->si_addr - sp; if (diff < 0) diff = -diff; if (diff < 4096) { - char msg[] = "error: stack overflow (possible infinite recursion)\n"; - [[gnu::unused]] auto res = write(2, msg, strlen(msg)); - _exit(1); // maybe abort instead? + nix::stackOverflowHandler(info, ctx); } } @@ -67,5 +66,12 @@ #endif } +std::function stackOverflowHandler(defaultStackOverflowHandler); + +void defaultStackOverflowHandler(siginfo_t * info, void * ctx) { + char msg[] = "error: stack overflow (possible infinite recursion)\n"; + [[gnu::unused]] auto res = write(2, msg, strlen(msg)); + _exit(1); // maybe abort instead? +} } diff -Nru nix-2.8.0/src/libstore/binary-cache-store.cc nix-2.16.1+dfsg/src/libstore/binary-cache-store.cc --- nix-2.8.0/src/libstore/binary-cache-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/binary-cache-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -9,7 +9,6 @@ #include "remote-fs-accessor.hh" #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" -#include "json.hh" #include "thread-pool.hh" #include "callback.hh" @@ -194,19 +193,12 @@ /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { - std::ostringstream jsonOut; - - { - JSONObject jsonRoot(jsonOut); - jsonRoot.attr("version", 1); - - { - auto res = jsonRoot.placeholder("root"); - listNar(res, ref(narAccessor), "", true); - } - } + nlohmann::json j = { + {"version", 1}, + {"root", listNar(ref(narAccessor), "", true)}, + }; - upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json"); + upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); } /* Optionally maintain an index of DWARF debug info files @@ -314,11 +306,22 @@ unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name, references), + *this, + name, + FixedOutputInfo { + .hash = { + .method = method, + .hash = nar.first, + }, + .references = { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, + }, nar.first, }; info.narSize = nar.second; - info.references = references; return info; })->path; } @@ -331,6 +334,17 @@ return fileExists(narInfoFileFor(storePath)); } +std::optional BinaryCacheStore::queryPathFromHashPart(const std::string & hashPart) +{ + auto pseudoPath = StorePath(hashPart + "-" + MissingName); + try { + auto info = queryPathInfo(pseudoPath); + return info->path; + } catch (InvalidPath &) { + return std::nullopt; + } +} + void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast(); @@ -343,7 +357,7 @@ try { getFile(info->url, *decompressor); } catch (NoSuchBinaryCacheFile & e) { - throw SubstituteGone(e.info()); + throw SubstituteGone(std::move(e.info())); } decompressor->finish(); @@ -367,7 +381,7 @@ auto callbackPtr = std::make_shared(std::move(callback)); getFile(narInfoFile, - {[=](std::future> fut) { + {[=,this](std::future> fut) { try { auto data = fut.get(); @@ -411,15 +425,22 @@ }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name, references), + *this, + name, + FixedOutputInfo { + .hash = { + .method = method, + .hash = h, + }, + .references = { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, + }, nar.first, }; info.narSize = nar.second; - info.references = references; - info.ca = FixedOutputHash { - .method = method, - .hash = h, - }; return info; })->path; } @@ -431,7 +452,7 @@ RepairFlag repair) { auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, textHash, references); + auto path = makeTextPath(name, TextInfo { { textHash }, references }); if (!repair && isValidPath(path)) return path; @@ -440,10 +461,16 @@ dumpString(s, sink); StringSource source(sink.s); return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { path, nar.first }; + ValidPathInfo info { + *this, + std::string { name }, + TextInfo { + { .hash = textHash }, + references, + }, + nar.first, + }; info.narSize = nar.second; - info.ca = TextHash { textHash }; - info.references = references; return info; })->path; } @@ -499,22 +526,9 @@ writeNarInfo(narInfo); } -std::optional BinaryCacheStore::getBuildLog(const StorePath & path) +std::optional BinaryCacheStore::getBuildLogExact(const StorePath & path) { - auto drvPath = path; - - if (!path.isDerivation()) { - try { - auto info = queryPathInfo(path); - // FIXME: add a "Log" field to .narinfo - if (!info->deriver) return std::nullopt; - drvPath = *info->deriver; - } catch (InvalidPath &) { - return std::nullopt; - } - } - - auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath))); + auto logPath = "log/" + std::string(baseNameOf(printStorePath(path))); debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); diff -Nru nix-2.8.0/src/libstore/binary-cache-store.hh nix-2.16.1+dfsg/src/libstore/binary-cache-store.hh --- nix-2.8.0/src/libstore/binary-cache-store.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/binary-cache-store.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "crypto.hh" #include "store-api.hh" @@ -16,19 +17,40 @@ { using StoreConfig::StoreConfig; - const Setting compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"}; - const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; - const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; - const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"}; - const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"}; + const Setting compression{(StoreConfig*) this, "xz", "compression", + "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; + + const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", + "Whether to write a JSON file that lists the files in each NAR."}; + + const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", + R"( + Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to + fetch debug info on demand + )"}; + + const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", + "Path to the secret key used to sign the binary cache."}; + + const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", + "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; + const Setting parallelCompression{(StoreConfig*) this, false, "parallel-compression", - "enable multi-threading compression for NARs, available for xz and zstd only currently"}; + "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; + const Setting compressionLevel{(StoreConfig*) this, -1, "compression-level", - "specify 'preset level' of compression to be used with NARs: " - "meaning and accepted range of values depends on compression method selected, " - "other than -1 which we reserve to indicate Nix defaults should be used"}; + R"( + The *preset level* to be used when compressing NARs. + The meaning and accepted values depend on the compression method selected. + `-1` specifies that the default compression level should be used. + )"}; }; + +/** + * @note subclasses must implement at least one of the two + * virtual getFile() methods. + */ class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store, public virtual LogStore @@ -58,14 +80,15 @@ std::string && data, const std::string & mimeType); - /* Note: subclasses must implement at least one of the two - following getFile() methods. */ - - /* Dump the contents of the specified file to a sink. */ + /** + * Dump the contents of the specified file to a sink. + */ virtual void getFile(const std::string & path, Sink & sink); - /* Fetch the specified file and call the specified callback with - the result. A subclass may implement this asynchronously. */ + /** + * Fetch the specified file and call the specified callback with + * the result. A subclass may implement this asynchronously. + */ virtual void getFile( const std::string & path, Callback> callback) noexcept; @@ -95,8 +118,7 @@ void queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept override; - std::optional queryPathFromHashPart(const std::string & hashPart) override - { unsupported("queryPathFromHashPart"); } + std::optional queryPathFromHashPart(const std::string & hashPart) override; void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) override; @@ -130,7 +152,7 @@ void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - std::optional getBuildLog(const StorePath & path) override; + std::optional getBuildLogExact(const StorePath & path) override; void addBuildLog(const StorePath & drvPath, std::string_view log) override; diff -Nru nix-2.8.0/src/libstore/build/derivation-goal.cc nix-2.16.1+dfsg/src/libstore/build/derivation-goal.cc --- nix-2.8.0/src/libstore/build/derivation-goal.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/derivation-goal.cc 2023-06-06 10:55:23.000000000 +0000 @@ -7,7 +7,6 @@ #include "finally.hh" #include "util.hh" #include "archive.hh" -#include "json.hh" #include "compression.hh" #include "worker-protocol.hh" #include "topo-sort.hh" @@ -40,7 +39,6 @@ #include #include #include -#include #include #include #include @@ -65,7 +63,7 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) , useDerivation(true) , drvPath(drvPath) @@ -84,7 +82,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) , useDerivation(false) , drvPath(drvPath) @@ -135,7 +133,7 @@ void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut, {}, ex); + done(BuildResult::TimedOut, {}, std::move(ex)); } @@ -144,18 +142,24 @@ (this->*state)(); } -void DerivationGoal::addWantedOutputs(const StringSet & outputs) +void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) { - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); - needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; + auto newWanted = wantedOutputs.union_(outputs); + switch (needRestart) { + case NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed: + if (!newWanted.isSubsetOf(wantedOutputs)) + needRestart = NeedRestartForMoreOutputs::OutputsAddedDoNeed; + break; + case NeedRestartForMoreOutputs::OutputsAddedDoNeed: + /* No need to check whether we added more outputs, because a + restart is already queued up. */ + break; + case NeedRestartForMoreOutputs::BuildInProgressWillNotNeed: + /* We are already building all outputs, so it doesn't matter if + we now want more. */ + break; + }; + wantedOutputs = newWanted; } @@ -207,10 +211,10 @@ parsedDrv = std::make_unique(drvPath, *drv); if (!drv->type().hasKnownOutputPaths()) - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); if (!drv->type().isPure()) { - settings.requireExperimentalFeature(Xp::ImpureDerivations); + experimentalFeatureSettings.require(Xp::ImpureDerivations); for (auto & [outputName, output] : drv->outputs) { auto randomPath = StorePath::random(outputPathName(drv->name, outputName)); @@ -270,11 +274,13 @@ ) ) ); - else + else { + auto * cap = getDerivationCA(*drv); addWaitee(upcast_goal(worker.makePathSubstitutionGoal( status.known->path, buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv)))); + cap ? std::optional { *cap } : std::nullopt))); + } } if (waitees.empty()) /* to prevent hang (no wake-up event) */ @@ -305,12 +311,29 @@ In particular, it may be the case that the hole in the closure is an output of the current derivation, which causes a loop if retried. */ - if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; + { + bool substitutionFailed = + nrIncompleteClosure > 0 && + nrIncompleteClosure == nrFailed; + switch (retrySubstitution) { + case RetrySubstitution::NoNeed: + if (substitutionFailed) + retrySubstitution = RetrySubstitution::YesNeed; + break; + case RetrySubstitution::YesNeed: + // Should not be able to reach this state from here. + assert(false); + break; + case RetrySubstitution::AlreadyRetried: + debug("substitution failed again, but we already retried once. Not retrying again."); + break; + } + } nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - if (needRestart) { - needRestart = false; + if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) { + needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; haveDerivation(); return; } @@ -338,13 +361,17 @@ produced using a substitute. So we have to build instead. */ void DerivationGoal::gaveUpOnSubstitution() { + /* At this point we are building all outputs, so if more are wanted there + is no need to restart. */ + needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed; + /* The inputs must be built before we can build this goal. */ inputDrvOutputs.clear(); if (useDerivation) for (auto & i : dynamic_cast(drv.get())->inputDrvs) { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ - if (drv->type().isPure() && !drv->type().isFixed()) { + if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { auto inputDrv = worker.evalStore.readDerivation(i.first); if (!inputDrv.type().isPure()) throw Error("pure derivation '%s' depends on impure derivation '%s'", @@ -392,7 +419,7 @@ auto outputs = queryDerivationOutputMap(); StorePathSet outputClosure; for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; + if (!wantedOutputs.contains(i.first)) continue; worker.store.computeFSClosure(i.second, outputClosure); } @@ -424,7 +451,7 @@ if (drvPath2 == outputsToDrv.end()) addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); + addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair)); } if (waitees.empty()) { @@ -459,8 +486,8 @@ return; } - if (retrySubstitution && !retriedSubstitution) { - retriedSubstitution = true; + if (retrySubstitution == RetrySubstitution::YesNeed) { + retrySubstitution = RetrySubstitution::AlreadyRetried; haveDerivation(); return; } @@ -485,7 +512,7 @@ ca.fixed /* Can optionally resolve if fixed, which is good for avoiding unnecessary rebuilds. */ - ? settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + ? experimentalFeatureSettings.isEnabled(Xp::CaDerivations) /* Must resolve if floating and there are any inputs drvs. */ : true); @@ -496,12 +523,20 @@ }, drvType.raw()); if (resolveDrv && !fullDrv.inputDrvs.empty()) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal. */ std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs); + if (!attempt) { + /* TODO (impure derivations-induced tech debt) (see below): + The above attempt should have found it, but because we manage + inputDrvOutputs statefully, sometimes it gets out of sync with + the real source of truth (store). So we query the store + directly if there's a problem. */ + attempt = fullDrv.tryResolve(worker.store); + } assert(attempt); Derivation drvResolved { *std::move(attempt) }; @@ -528,13 +563,32 @@ /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - for (auto & j : wantedDepOutputs) - if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) + for (auto & j : wantedDepOutputs) { + /* TODO (impure derivations-induced tech debt): + Tracking input derivation outputs statefully through the + goals is error prone and has led to bugs. + For a robust nix, we need to move towards the `else` branch, + which does not rely on goal state to match up with the + reality of the store, which is our real source of truth. + However, the impure derivations feature still relies on this + fragile way of doing things, because its builds do not have + a representation in the store, which is a usability problem + in itself. When implementing this logic entirely with lookups + make sure that they're cached. */ + if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) { worker.store.computeFSClosure(*outPath, inputPaths); - else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + } + else { + auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath); + auto outMapPath = outMap.find(j); + if (outMapPath == outMap.end()) { + throw Error( + "derivation '%s' requires non-existent output '%s' from input derivation '%s'", + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + } + worker.store.computeFSClosure(outMapPath->second, inputPaths); + } + } } } @@ -546,17 +600,11 @@ /* What type of derivation are we building? */ derivationType = drv->type(); - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationType.isFixed() ? 1 : settings.buildRepeat + 1; - /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); - - buildResult = BuildResult { .path = buildResult.path }; } void DerivationGoal::started() @@ -564,12 +612,11 @@ auto msg = fmt( buildMode == bmRepair ? "repairing outputs of '%s'" : buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); + "building '%s'", worker.store.printStorePath(drvPath)); fmt("building '%s'", worker.store.printStorePath(drvPath)); if (hook) msg += fmt(" on '%s'", machineName); act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); + Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", 1, 1}); mcRunningBuilds = std::make_unique>(worker.runningBuilds); worker.updateProgress(); } @@ -705,8 +752,7 @@ if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); + renameFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -719,7 +765,7 @@ tmpPath (the replacement), so we have to move it out of the way first. We'd better not be interrupted here, because if we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); + Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), random()); if (pathExists(storePath)) movePath(storePath, oldPath); @@ -786,8 +832,7 @@ Store & store, Logger & logger, const StorePath & drvPath, - StorePathSet outputPaths -) + const StorePathSet & outputPaths) { auto hook = settings.postBuildHook; if (hook == "") @@ -871,6 +916,14 @@ cleanupPostChildKill(); + if (buildResult.cpuUser && buildResult.cpuSystem) { + debug("builder for '%s' terminated with status %d, user CPU %.3fs, system CPU %.3fs", + worker.store.printStorePath(drvPath), + status, + ((double) buildResult.cpuUser->count()) / 1000000, + ((double) buildResult.cpuSystem->count()) / 1000000); + } + bool diskFull = false; try { @@ -891,7 +944,11 @@ msg += line; msg += "\n"; } - msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.", + auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand) + ? "nix log" + : "nix-store -l"; + msg += fmt("For full logs, run '" ANSI_BOLD "%s %s" ANSI_NORMAL "'.", + nixLogCommand, worker.store.printStorePath(drvPath)); } @@ -906,7 +963,7 @@ auto builtOutputs = registerOutputs(); StorePathSet outputPaths; - for (auto & [_, output] : buildResult.builtOutputs) + for (auto & [_, output] : builtOutputs) outputPaths.insert(output.outPath); runPostBuildHook( worker.store, @@ -915,22 +972,8 @@ outputPaths ); - if (buildMode == bmCheck) { - cleanupPostOutputsRegisteredModeCheck(); - done(BuildResult::Built, std::move(builtOutputs)); - return; - } - cleanupPostOutputsRegisteredModeNonCheck(); - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will not create new lock files with the same names as the old @@ -959,7 +1002,7 @@ BuildResult::PermanentFailure; } - done(st, {}, e); + done(st, {}, std::move(e)); return; } } @@ -972,26 +1015,40 @@ auto resolvedDrv = *resolvedDrvGoal->drv; auto & resolvedResult = resolvedDrvGoal->buildResult; - DrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs; if (resolvedResult.success()) { auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); StorePathSet outputPaths; - // `wantedOutputs` might be empty, which means “all the outputs” - auto realWantedOutputs = wantedOutputs; - if (realWantedOutputs.empty()) - realWantedOutputs = resolvedDrv.outputNames(); - - for (auto & wantedOutput : realWantedOutputs) { - assert(initialOutputs.count(wantedOutput) != 0); - assert(resolvedHashes.count(wantedOutput) != 0); - auto realisation = resolvedResult.builtOutputs.at( - DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput }); + for (auto & outputName : resolvedDrv.outputNames()) { + auto initialOutput = get(initialOutputs, outputName); + auto resolvedHash = get(resolvedHashes, outputName); + if ((!initialOutput) || (!resolvedHash)) + throw Error( + "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)", + worker.store.printStorePath(drvPath), outputName); + + auto realisation = [&]{ + auto take1 = get(resolvedResult.builtOutputs, outputName); + if (take1) return *take1; + + /* The above `get` should work. But sateful tracking of + outputs in resolvedResult, this can get out of sync with the + store, which is our actual source of truth. For now we just + check the store directly if it fails. */ + auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, outputName }); + if (take2) return *take2; + + throw Error( + "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", + worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName); + }(); + if (drv->type().isPure()) { auto newRealisation = realisation; - newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput }; + newRealisation.id = DrvOutput { initialOutput->outputHash, outputName }; newRealisation.signatures.clear(); if (!drv->type().isFixed()) newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath); @@ -999,7 +1056,7 @@ worker.store.registerDrvOutput(newRealisation); } outputPaths.insert(realisation.outPath); - builtOutputs.emplace(realisation.id, realisation); + builtOutputs.emplace(outputName, realisation); } runPostBuildHook( @@ -1095,7 +1152,7 @@ /* Tell the hook all the inputs that have to be copied to the remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); + workerProtoWrite(worker.store, hook->sink, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1106,7 +1163,7 @@ if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); } - worker_proto::write(worker.store, hook->sink, missingOutputs); + workerProtoWrite(worker.store, hook->sink, missingOutputs); } hook->sink = FdSink(); @@ -1124,7 +1181,7 @@ } -DrvOutputs DerivationGoal::registerOutputs() +SingleDrvOutputs DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -1286,17 +1343,28 @@ } -std::pair DerivationGoal::checkPathValidity() +std::pair DerivationGoal::checkPathValidity() { if (!drv->type().isPure()) return { false, {} }; bool checkHash = buildMode == bmRepair; - auto wantedOutputsLeft = wantedOutputs; - DrvOutputs validOutputs; + auto wantedOutputsLeft = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return StringSet {}; + }, + [&](const OutputsSpec::Names & names) { + return static_cast(names); + }, + }, wantedOutputs.raw()); + SingleDrvOutputs validOutputs; for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput & info = initialOutputs.at(i.first); - info.wanted = wantOutput(i.first, wantedOutputs); + auto initialOutput = get(initialOutputs, i.first); + if (!initialOutput) + // this is an invalid output, gets catched with (!wantedOutputsLeft.empty()) + continue; + auto & info = *initialOutput; + info.wanted = wantedOutputs.contains(i.first); if (info.wanted) wantedOutputsLeft.erase(i.first); if (i.second) { @@ -1310,8 +1378,8 @@ : PathStatus::Corrupt, }; } - auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first}; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + auto drvOutput = DrvOutput{info.outputHash, i.first}; + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { .path = real->outPath, @@ -1330,11 +1398,11 @@ ); } } - if (info.wanted && info.known && info.known->isValid()) - validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); + if (info.known && info.known->isValid()) + validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path }); } - // If we requested all the outputs via the empty set, we are always fine. + // If we requested all the outputs, we are always fine. // If we requested specific elements, the loop above removes all the valid // ones, so any that are left must be invalid. if (!wantedOutputsLeft.empty()) @@ -1355,7 +1423,7 @@ } -DrvOutputs DerivationGoal::assertPathValidity() +SingleDrvOutputs DerivationGoal::assertPathValidity() { auto [allValid, validOutputs] = checkPathValidity(); if (!allValid) @@ -1366,7 +1434,7 @@ void DerivationGoal::done( BuildResult::Status status, - DrvOutputs builtOutputs, + SingleDrvOutputs builtOutputs, std::optional ex) { buildResult.status = status; @@ -1381,8 +1449,9 @@ mcRunningBuilds.reset(); if (buildResult.success()) { - assert(!builtOutputs.empty()); - buildResult.builtOutputs = std::move(builtOutputs); + auto wantedBuiltOutputs = filterDrvOutputs(wantedOutputs, std::move(builtOutputs)); + assert(!wantedBuiltOutputs.empty()); + buildResult.builtOutputs = std::move(wantedBuiltOutputs); if (status == BuildResult::Built) worker.doneBuilds++; } else { @@ -1399,7 +1468,7 @@ fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } - amDone(buildResult.success() ? ecSuccess : ecFailed, ex); + amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } @@ -1407,12 +1476,28 @@ { Goal::waiteeDone(waitee, result); - if (waitee->buildResult.success()) - if (auto bfd = std::get_if(&waitee->buildResult.path)) - for (auto & [output, realisation] : waitee->buildResult.builtOutputs) + if (!useDerivation) return; + auto & fullDrv = *dynamic_cast(drv.get()); + + auto * dg = dynamic_cast(&*waitee); + if (!dg) return; + + auto outputs = fullDrv.inputDrvs.find(dg->drvPath); + if (outputs == fullDrv.inputDrvs.end()) return; + + for (auto & outputName : outputs->second) { + auto buildResult = dg->getBuildResult(DerivedPath::Built { + .drvPath = dg->drvPath, + .outputs = OutputsSpec::Names { outputName }, + }); + if (buildResult.success()) { + auto i = buildResult.builtOutputs.find(outputName); + if (i != buildResult.builtOutputs.end()) inputDrvOutputs.insert_or_assign( - { bfd->drvPath, output.outputName }, - realisation.outPath); + { dg->drvPath, outputName }, + i->second.outPath); + } + } } } diff -Nru nix-2.8.0/src/libstore/build/derivation-goal.hh nix-2.16.1+dfsg/src/libstore/build/derivation-goal.hh --- nix-2.8.0/src/libstore/build/derivation-goal.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/derivation-goal.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,7 +1,9 @@ #pragma once +///@file #include "parsed-derivations.hh" #include "lock.hh" +#include "outputs-spec.hh" #include "store-api.hh" #include "pathlocks.hh" #include "goal.hh" @@ -14,8 +16,10 @@ typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ +/** + * Unless we are repairing, we don't both to test validity and just assume it, + * so the choices are `Absent` or `Valid`. + */ enum struct PathStatus { Corrupt, Absent, @@ -25,11 +29,15 @@ struct InitialOutputStatus { StorePath path; PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ + /** + * Valid in the store, and additionally non-corrupt if we are repairing + */ bool isValid() const { return status == PathStatus::Valid; } - /* Merely present, allowed to be corrupt */ + /** + * Merely present, allowed to be corrupt + */ bool isPresent() const { return status == PathStatus::Corrupt || status == PathStatus::Valid; @@ -44,59 +52,123 @@ struct DerivationGoal : public Goal { - /* Whether to use an on-disk .drv file. */ + /** + * Whether to use an on-disk .drv file. + */ bool useDerivation; - /* The path of the derivation. */ + /** The path of the derivation. */ StorePath drvPath; - /* The goal for the corresponding resolved derivation */ + /** + * The goal for the corresponding resolved derivation + */ std::shared_ptr resolvedDrvGoal; - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Mapping from input derivations + output names to actual store - paths. This is filled in by waiteeDone() as each dependency - finishes, before inputsRealised() is reached, */ + /** + * The specific outputs that we need to build. Empty means all of + * them. + */ + OutputsSpec wantedOutputs; + + /** + * Mapping from input derivations + output names to actual store + * paths. This is filled in by waiteeDone() as each dependency + * finishes, before inputsRealised() is reached. + */ std::map, StorePath> inputDrvOutputs; - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. This is done in case of an incomplete closure. */ - bool retrySubstitution = false; - - /* Whether we've retried substitution, in which case we won't try - again. */ - bool retriedSubstitution = false; - - /* The derivation stored at drvPath. */ + /** + * See `needRestart`; just for that field. + */ + enum struct NeedRestartForMoreOutputs { + /** + * The goal state machine is progressing based on the current value of + * `wantedOutputs. No actions are needed. + */ + OutputsUnmodifedDontNeed, + /** + * `wantedOutputs` has been extended, but the state machine is + * proceeding according to its old value, so we need to restart. + */ + OutputsAddedDoNeed, + /** + * The goal state machine has progressed to the point of doing a build, + * in which case all outputs will be produced, so extensions to + * `wantedOutputs` no longer require a restart. + */ + BuildInProgressWillNotNeed, + }; + + /** + * Whether additional wanted outputs have been added. + */ + NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; + + /** + * See `retrySubstitution`; just for that field. + */ + enum RetrySubstitution { + /** + * No issues have yet arose, no need to restart. + */ + NoNeed, + /** + * Something failed and there is an incomplete closure. Let's retry + * substituting. + */ + YesNeed, + /** + * We are current or have already retried substitution, and whether or + * not something goes wrong we will not retry again. + */ + AlreadyRetried, + }; + + /** + * Whether to retry substituting the outputs after building the + * inputs. This is done in case of an incomplete closure. + */ + RetrySubstitution retrySubstitution = RetrySubstitution::NoNeed; + + /** + * The derivation stored at drvPath. + */ std::unique_ptr drv; std::unique_ptr parsedDrv; - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ + /** + * The remainder is state held during the build. + */ + + /** + * Locks on (fixed) output paths. + */ PathLocks outputLocks; - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ + /** + * All input paths (that is, the union of FS closures of the + * immediate input paths). + */ StorePathSet inputPaths; std::map initialOutputs; - /* File descriptor for the log file. */ + /** + * File descriptor for the log file. + */ AutoCloseFD fdLogFile; std::shared_ptr logFileSink, logSink; - /* Number of bytes received from the builder's stdout/stderr. */ + /** + * Number of bytes received from the builder's stdout/stderr. + */ unsigned long logSize; - /* The most recent log lines. */ + /** + * The most recent log lines. + */ std::list logTail; std::string currentLogLine; @@ -104,10 +176,14 @@ std::string currentHookLine; - /* The build hook. */ + /** + * The build hook. + */ std::unique_ptr hook; - /* The sort of derivation we are building. */ + /** + * The sort of derivation we are building. + */ DerivationType derivationType; typedef void (DerivationGoal::*GoalState)(); @@ -115,28 +191,27 @@ BuildMode buildMode; - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; std::unique_ptr act; - /* Activity that denotes waiting for a lock. */ + /** + * Activity that denotes waiting for a lock. + */ std::unique_ptr actLock; std::map builderActivities; - /* The remote machine on which we're building. */ + /** + * The remote machine on which we're building. + */ std::string machineName; DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); virtual ~DerivationGoal(); @@ -146,10 +221,14 @@ void work() override; - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - /* The states. */ + /** + * Add wanted outputs to an already existing derivation goal. + */ + void addWantedOutputs(const OutputsSpec & outputs); + + /** + * The states. + */ void getDerivation(); void loadDerivation(); void haveDerivation(); @@ -163,28 +242,42 @@ void resolvedFinished(); - /* Is the build hook willing to perform the build? */ + /** + * Is the build hook willing to perform the build? + */ HookReply tryBuildHook(); virtual int getChildStatus(); - /* Check that the derivation outputs all exist and register them - as valid. */ - virtual DrvOutputs registerOutputs(); - - /* Open a log file and a pipe to it. */ + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ + virtual SingleDrvOutputs registerOutputs(); + + /** + * Open a log file and a pipe to it. + */ Path openLogFile(); - /* Sign the newly built realisation if the store allows it */ + /** + * Sign the newly built realisation if the store allows it + */ virtual void signRealisation(Realisation&) {} - /* Close the log file. */ + /** + * Close the log file. + */ void closeLogFile(); - /* Close the read side of the logger pipe. */ + /** + * Close the read side of the logger pipe. + */ virtual void closeReadPipes(); - /* Cleanup hooks for buildDone() */ + /** + * Cleanup hooks for buildDone() + */ virtual void cleanupHookFinally(); virtual void cleanupPreChildKill(); virtual void cleanupPostChildKill(); @@ -194,30 +287,38 @@ virtual bool isReadDesc(int fd); - /* Callback used by the worker to write to the log. */ + /** + * Callback used by the worker to write to the log. + */ void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; void flushLine(); - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ + /** + * Wrappers around the corresponding Store methods that first consult the + * derivation. This is currently needed because when there is no drv file + * there also is no DB entry. + */ std::map> queryPartialDerivationOutputMap(); OutputPathMap queryDerivationOutputMap(); - /* Update 'initialOutputs' to determine the current status of the - outputs of the derivation. Also returns a Boolean denoting - whether all outputs are valid and non-corrupt, and a - 'DrvOutputs' structure containing the valid and wanted - outputs. */ - std::pair checkPathValidity(); - - /* Aborts if any output is not valid or corrupt, and otherwise - returns a 'DrvOutputs' structure containing the wanted - outputs. */ - DrvOutputs assertPathValidity(); - - /* Forcibly kill the child process, if any. */ + /** + * Update 'initialOutputs' to determine the current status of the + * outputs of the derivation. Also returns a Boolean denoting + * whether all outputs are valid and non-corrupt, and a + * 'SingleDrvOutputs' structure containing the valid outputs. + */ + std::pair checkPathValidity(); + + /** + * Aborts if any output is not valid or corrupt, and otherwise + * returns a 'SingleDrvOutputs' structure containing all outputs. + */ + SingleDrvOutputs assertPathValidity(); + + /** + * Forcibly kill the child process, if any. + */ virtual void killChild(); void repairClosure(); @@ -226,12 +327,14 @@ void done( BuildResult::Status status, - DrvOutputs builtOutputs = {}, + SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); void waiteeDone(GoalPtr waitee, ExitCode result) override; StorePathSet exportReferences(const StorePathSet & storePaths); + + JobCategory jobCategory() override { return JobCategory::Build; }; }; MakeError(NotDeterministic, BuildError); diff -Nru nix-2.8.0/src/libstore/build/drv-output-substitution-goal.cc nix-2.16.1+dfsg/src/libstore/build/drv-output-substitution-goal.cc --- nix-2.8.0/src/libstore/build/drv-output-substitution-goal.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/drv-output-substitution-goal.cc 2023-06-06 10:55:23.000000000 +0000 @@ -61,20 +61,25 @@ // FIXME: Make async // outputInfo = sub->queryRealisation(id); - outPipe.create(); - promise = decltype(promise)(); + + /* The callback of the curl download below can outlive `this` (if + some other error occurs), so it must not touch `this`. So put + the shared state in a separate refcounted object. */ + downloadState = std::make_shared(); + downloadState->outPipe.create(); sub->queryRealisation( - id, { [&](std::future> res) { + id, + { [downloadState(downloadState)](std::future> res) { try { - Finally updateStats([this]() { outPipe.writeSide.close(); }); - promise.set_value(res.get()); + Finally updateStats([&]() { downloadState->outPipe.writeSide.close(); }); + downloadState->promise.set_value(res.get()); } catch (...) { - promise.set_exception(std::current_exception()); + downloadState->promise.set_exception(std::current_exception()); } } }); - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); + worker.childStarted(shared_from_this(), {downloadState->outPipe.readSide.get()}, true, false); state = &DrvOutputSubstitutionGoal::realisationFetched; } @@ -84,7 +89,7 @@ worker.childTerminated(this); try { - outputInfo = promise.get_future().get(); + outputInfo = downloadState->promise.get_future().get(); } catch (std::exception & e) { printError(e.what()); substituterFailed = true; @@ -155,7 +160,7 @@ void DrvOutputSubstitutionGoal::handleEOF(int fd) { - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); + if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } diff -Nru nix-2.8.0/src/libstore/build/drv-output-substitution-goal.hh nix-2.16.1+dfsg/src/libstore/build/drv-output-substitution-goal.hh --- nix-2.8.0/src/libstore/build/drv-output-substitution-goal.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/drv-output-substitution-goal.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "goal.hh" @@ -10,31 +11,47 @@ class Worker; -// Substitution of a derivation output. -// This is done in three steps: -// 1. Fetch the output info from a substituter -// 2. Substitute the corresponding output path -// 3. Register the output info +/** + * Substitution of a derivation output. + * This is done in three steps: + * 1. Fetch the output info from a substituter + * 2. Substitute the corresponding output path + * 3. Register the output info + */ class DrvOutputSubstitutionGoal : public Goal { -private: - // The drv output we're trying to substitue + + /** + * The drv output we're trying to substitute + */ DrvOutput id; - // The realisation corresponding to the given output id. - // Will be filled once we can get it. + /** + * The realisation corresponding to the given output id. + * Will be filled once we can get it. + */ std::shared_ptr outputInfo; - /* The remaining substituters. */ + /** + * The remaining substituters. + */ std::list> subs; - /* The current substituter. */ + /** + * The current substituter. + */ std::shared_ptr sub; - Pipe outPipe; - std::thread thr; - std::promise> promise; - - /* Whether a substituter failed. */ + struct DownloadState + { + Pipe outPipe; + std::promise> promise; + }; + + std::shared_ptr downloadState; + + /** + * Whether a substituter failed. + */ bool substituterFailed = false; public: @@ -55,6 +72,8 @@ void work() override; void handleEOF(int fd) override; + + JobCategory jobCategory() override { return JobCategory::Substitution; }; }; } diff -Nru nix-2.8.0/src/libstore/build/entry-points.cc nix-2.16.1+dfsg/src/libstore/build/entry-points.cc --- nix-2.8.0/src/libstore/build/entry-points.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/entry-points.cc 2023-06-06 10:55:23.000000000 +0000 @@ -10,16 +10,8 @@ Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; - for (const auto & br : reqs) { - std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); - }, - [&](const DerivedPath::Opaque & bo) { - goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); - }, - }, br.raw()); - } + for (auto & br : reqs) + goals.insert(worker.makeGoal(br, buildMode)); worker.run(goals); @@ -30,7 +22,7 @@ if (ex) logError(i->ex->info()); else - ex = i->ex; + ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); @@ -40,14 +32,14 @@ if (failed.size() == 1 && ex) { ex->status = worker.exitStatus(); - throw *ex; + throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); } } -std::vector Store::buildPathsWithResults( +std::vector Store::buildPathsWithResults( const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) @@ -55,23 +47,23 @@ Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; - for (const auto & br : reqs) { - std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); - }, - [&](const DerivedPath::Opaque & bo) { - goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); - }, - }, br.raw()); + std::vector> state; + + for (const auto & req : reqs) { + auto goal = worker.makeGoal(req, buildMode); + goals.insert(goal); + state.push_back({req, goal}); } worker.run(goals); - std::vector results; + std::vector results; - for (auto & i : goals) - results.push_back(i->buildResult); + for (auto & [req, goalPtr] : state) + results.emplace_back(KeyedBuildResult { + goalPtr->getBuildResult(req), + /* .path = */ req, + }); return results; } @@ -80,16 +72,18 @@ BuildMode buildMode) { Worker worker(*this, *this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode); try { worker.run(Goals{goal}); - return goal->buildResult; + return goal->getBuildResult(DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All {}, + }); } catch (Error & e) { return BuildResult { .status = BuildResult::MiscFailure, .errorMsg = e.msg(), - .path = DerivedPath::Built { .drvPath = drvPath }, }; }; } @@ -109,14 +103,14 @@ if (goal->exitCode != Goal::ecSuccess) { if (goal->ex) { goal->ex->status = worker.exitStatus(); - throw *goal->ex; + throw std::move(*goal->ex); } else throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } } -void LocalStore::repairPath(const StorePath & path) +void Store::repairPath(const StorePath & path) { Worker worker(*this, *this); GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair); @@ -130,7 +124,8 @@ auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { goals.clear(); - goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); + // FIXME: Should just build the specific output we need. + goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair)); worker.run(goals); } else throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); diff -Nru nix-2.8.0/src/libstore/build/goal.cc nix-2.16.1+dfsg/src/libstore/build/goal.cc --- nix-2.8.0/src/libstore/build/goal.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/goal.cc 2023-06-06 10:55:23.000000000 +0000 @@ -11,6 +11,29 @@ } +BuildResult Goal::getBuildResult(const DerivedPath & req) { + BuildResult res { buildResult }; + + if (auto pbp = std::get_if(&req)) { + auto & bp = *pbp; + + /* Because goals are in general shared between derived paths + that share the same derivation, we need to filter their + results to get back just the results we care about. + */ + + for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) { + if (bp.outputs.contains(it->first)) + ++it; + else + it = res.builtOutputs.erase(it); + } + } + + return res; +} + + void addToWeakGoals(WeakGoals & goals, GoalPtr p) { if (goals.find(p) != goals.end()) @@ -78,9 +101,9 @@ } -void Goal::trace(const FormatOrString & fs) +void Goal::trace(std::string_view s) { - debug("%1%: %2%", name, fs.s); + debug("%1%: %2%", name, s); } } diff -Nru nix-2.8.0/src/libstore/build/goal.hh nix-2.16.1+dfsg/src/libstore/build/goal.hh --- nix-2.8.0/src/libstore/build/goal.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/goal.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "store-api.hh" @@ -6,11 +7,15 @@ namespace nix { -/* Forward definition. */ +/** + * Forward definition. + */ struct Goal; class Worker; -/* A pointer to a goal. */ +/** + * A pointer to a goal. + */ typedef std::shared_ptr GoalPtr; typedef std::weak_ptr WeakGoalPtr; @@ -18,53 +23,102 @@ bool operator() (const GoalPtr & a, const GoalPtr & b) const; }; -/* Set of goals. */ +/** + * Set of goals. + */ typedef std::set Goals; typedef std::set> WeakGoals; -/* A map of paths to goals (and the other way around). */ +/** + * A map of paths to goals (and the other way around). + */ typedef std::map WeakGoalMap; +/** + * Used as a hint to the worker on how to schedule a particular goal. For example, + * builds are typically CPU- and memory-bound, while substitutions are I/O bound. + * Using this information, the worker might decide to schedule more or fewer goals + * of each category in parallel. + */ +enum struct JobCategory { + Build, + Substitution, +}; + struct Goal : public std::enable_shared_from_this { typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - /* Backlink to the worker. */ + /** + * Backlink to the worker. + */ Worker & worker; - /* Goals that this goal is waiting for. */ + /** + * Goals that this goal is waiting for. + */ Goals waitees; - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ + /** + * Goals waiting for this one to finish. Must use weak pointers + * here to prevent cycles. + */ WeakGoals waiters; - /* Number of goals we are/were waiting for that have failed. */ + /** + * Number of goals we are/were waiting for that have failed. + */ size_t nrFailed = 0; - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because there are no substituters. + */ size_t nrNoSubstituters = 0; - /* Number of substitution goals we are/were waiting for that - failed because they had unsubstitutable references. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because they had unsubstitutable references. + */ size_t nrIncompleteClosure = 0; - /* Name of this goal for debugging purposes. */ + /** + * Name of this goal for debugging purposes. + */ std::string name; - /* Whether the goal is finished. */ + /** + * Whether the goal is finished. + */ ExitCode exitCode = ecBusy; - /* Build result. */ +protected: + /** + * Build result. + */ BuildResult buildResult; - /* Exception containing an error message, if any. */ +public: + + /** + * Project a `BuildResult` with just the information that pertains + * to the given request. + * + * In general, goals may be aliased between multiple requests, and + * the stored `BuildResult` has information for the union of all + * requests. We don't want to leak what the other request are for + * sake of both privacy and determinism, and this "safe accessor" + * ensures we don't. + */ + BuildResult getBuildResult(const DerivedPath &); + + /** + * Exception containing an error message, if any. + */ std::optional ex; Goal(Worker & worker, DerivedPath path) : worker(worker) - , buildResult { .path = std::move(path) } { } virtual ~Goal() @@ -88,16 +142,18 @@ abort(); } - void trace(const FormatOrString & fs); + void trace(std::string_view s); std::string getName() { return name; } - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ + /** + * Callback in case of a timeout. It should wake up its waiters, + * get rid of any running child processes that are being monitored + * by the worker (important!), etc. + */ virtual void timedOut(Error && ex) = 0; virtual std::string key() = 0; @@ -105,6 +161,8 @@ void amDone(ExitCode result, std::optional ex = {}); virtual void cleanup() { } + + virtual JobCategory jobCategory() = 0; }; void addToWeakGoals(WeakGoals & goals, GoalPtr p); diff -Nru nix-2.8.0/src/libstore/build/hook-instance.cc nix-2.16.1+dfsg/src/libstore/build/hook-instance.cc --- nix-2.8.0/src/libstore/build/hook-instance.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/hook-instance.cc 2023-06-06 10:55:23.000000000 +0000 @@ -7,6 +7,22 @@ { debug("starting build hook '%s'", settings.buildHook); + auto buildHookArgs = tokenizeString>(settings.buildHook.get()); + + if (buildHookArgs.empty()) + throw Error("'build-hook' setting is empty"); + + auto buildHook = buildHookArgs.front(); + buildHookArgs.pop_front(); + + Strings args; + args.push_back(std::string(baseNameOf(buildHook))); + + for (auto & arg : buildHookArgs) + args.push_back(arg); + + args.push_back(std::to_string(verbosity)); + /* Create a pipe to get the output of the child. */ fromHook.create(); @@ -19,7 +35,10 @@ /* Fork the hook. */ pid = startProcess([&]() { - commonChildInit(fromHook); + if (dup2(fromHook.writeSide.get(), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + + commonChildInit(); if (chdir("/") == -1) throw SysError("changing into /"); @@ -36,14 +55,9 @@ if (dup2(builderOut.readSide.get(), 5) == -1) throw SysError("dupping builder's stdout/stderr"); - Strings args = { - std::string(baseNameOf(settings.buildHook.get())), - std::to_string(verbosity), - }; - - execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); + execv(buildHook.c_str(), stringsToCharPtrs(args).data()); - throw SysError("executing '%s'", settings.buildHook); + throw SysError("executing '%s'", buildHook); }); pid.setSeparatePG(true); diff -Nru nix-2.8.0/src/libstore/build/hook-instance.hh nix-2.16.1+dfsg/src/libstore/build/hook-instance.hh --- nix-2.8.0/src/libstore/build/hook-instance.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/hook-instance.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "logging.hh" #include "serialise.hh" @@ -7,16 +8,24 @@ struct HookInstance { - /* Pipes for talking to the build hook. */ + /** + * Pipes for talking to the build hook. + */ Pipe toHook; - /* Pipe for the hook's standard output/error. */ + /** + * Pipe for the hook's standard output/error. + */ Pipe fromHook; - /* Pipe for the builder's standard output/error. */ + /** + * Pipe for the builder's standard output/error. + */ Pipe builderOut; - /* The process ID of the hook. */ + /** + * The process ID of the hook. + */ Pid pid; FdSink sink; diff -Nru nix-2.8.0/src/libstore/build/local-derivation-goal.cc nix-2.16.1+dfsg/src/libstore/build/local-derivation-goal.cc --- nix-2.8.0/src/libstore/build/local-derivation-goal.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/local-derivation-goal.cc 2023-06-06 10:55:23.000000000 +0000 @@ -8,12 +8,15 @@ #include "finally.hh" #include "util.hh" #include "archive.hh" -#include "json.hh" #include "compression.hh" #include "daemon.hh" #include "worker-protocol.hh" #include "topo-sort.hh" #include "callback.hh" +#include "json-utils.hh" +#include "cgroup.hh" +#include "personality.hh" +#include "namespaces.hh" #include #include @@ -23,7 +26,6 @@ #include #include #include -#include #include #include @@ -36,7 +38,6 @@ #include #include #include -#include #include #include #include @@ -55,8 +56,7 @@ #include #include - -#include +#include namespace nix { @@ -130,27 +130,46 @@ if (pid != -1) { worker.childTerminated(this); - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); + /* If we're using a build user, then there is a tricky race + condition: if we kill the build user before the child has + done its setuid() to the build user uid, then it won't be + killed, and we'll potentially lock up in pid.wait(). So + also send a conventional kill to the child. */ + ::kill(-pid, SIGKILL); /* ignore the result */ + + killSandbox(true); - assert(pid == -1); + pid.wait(); } DerivationGoal::killChild(); } -void LocalDerivationGoal::tryLocalBuild() { +void LocalDerivationGoal::killSandbox(bool getStats) +{ + if (cgroup) { + #if __linux__ + auto stats = destroyCgroup(*cgroup); + if (getStats) { + buildResult.cpuUser = stats.cpuUser; + buildResult.cpuSystem = stats.cpuSystem; + } + #else + abort(); + #endif + } + + else if (buildUser) { + auto uid = buildUser->getUID(); + assert(uid != 0); + killUser(uid); + } +} + + +void LocalDerivationGoal::tryLocalBuild() +{ unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { state = &DerivationGoal::tryToBuild; @@ -159,28 +178,57 @@ return; } - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); - - if (buildUser->findFreeUser()) { - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } else { + /* Are we doing a chroot build? */ + { + auto noChroot = parsedDrv->getBoolAttr("__noChroot"); + if (settings.sandboxMode == smEnabled) { + if (noChroot) + throw Error("derivation '%s' has '__noChroot' set, " + "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); +#if __APPLE__ + if (additionalSandboxProfile != "") + throw Error("derivation '%s' specifies a sandbox profile, " + "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); +#endif + useChroot = true; + } + else if (settings.sandboxMode == smDisabled) + useChroot = false; + else if (settings.sandboxMode == smRelaxed) + useChroot = derivationType.isSandboxed() && !noChroot; + } + + auto & localStore = getLocalStore(); + if (localStore.storeDir != localStore.realStoreDir.get()) { + #if __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif + } + + #if __linux__ + if (useChroot) { + if (!mountAndPidNamespacesSupported()) { + if (!settings.sandboxFallback) + throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); + debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); + useChroot = false; + } + } + #endif + + if (useBuildUsers()) { + if (!buildUser) + buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot); + + if (!buildUser) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); return; } -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif } actLock.reset(); @@ -194,7 +242,7 @@ outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, {}, e); + done(BuildResult::InputRejected, {}, std::move(e)); return; } @@ -224,8 +272,7 @@ if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); + renameFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -245,7 +292,7 @@ if (hook) { DerivationGoal::closeReadPipes(); } else - builderOut.readSide = -1; + builderOut.close(); } @@ -272,7 +319,7 @@ malicious user from leaving behind a process that keeps files open and modifies them after they have been chown'ed to root. */ - if (buildUser) buildUser->kill(); + killSandbox(true); /* Terminate the recursive Nix daemon. */ stopDaemon(); @@ -310,9 +357,9 @@ for (auto & [_, status] : initialOutputs) { if (!status.known) continue; if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); + auto p = worker.store.toRealPath(status.known->path); if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); + renameFile((chrootRootDir + p), p); } return diskFull; @@ -338,12 +385,6 @@ } -int childEntry(void * arg) -{ - ((LocalDerivationGoal *) arg)->runChild(); - return 1; -} - #if __linux__ static void linkOrCopy(const Path & from, const Path & to) { @@ -365,6 +406,64 @@ void LocalDerivationGoal::startBuilder() { + if ((buildUser && buildUser->getUIDCount() != 1) + #if __linux__ + || settings.useCgroups + #endif + ) + { + #if __linux__ + experimentalFeatureSettings.require(Xp::Cgroups); + + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) + throw Error("cannot determine the cgroups file system"); + + auto ourCgroups = getCgroups("/proc/self/cgroup"); + auto ourCgroup = ourCgroups[""]; + if (ourCgroup == "") + throw Error("cannot determine cgroup name from /proc/self/cgroup"); + + auto ourCgroupPath = canonPath(*cgroupFS + "/" + ourCgroup); + + if (!pathExists(ourCgroupPath)) + throw Error("expected cgroup directory '%s'", ourCgroupPath); + + static std::atomic counter{0}; + + cgroup = buildUser + ? fmt("%s/nix-build-uid-%d", ourCgroupPath, buildUser->getUID()) + : fmt("%s/nix-build-pid-%d-%d", ourCgroupPath, getpid(), counter++); + + debug("using cgroup '%s'", *cgroup); + + /* When using a build user, record the cgroup we used for that + user so that if we got interrupted previously, we can kill + any left-over cgroup first. */ + if (buildUser) { + auto cgroupsDir = settings.nixStateDir + "/cgroups"; + createDirs(cgroupsDir); + + auto cgroupFile = fmt("%s/%d", cgroupsDir, buildUser->getUID()); + + if (pathExists(cgroupFile)) { + auto prevCgroup = readFile(cgroupFile); + destroyCgroup(prevCgroup); + } + + writeFile(cgroupFile, *cgroup); + } + + #else + throw Error("cgroups are not supported on this platform"); + #endif + } + + /* Make sure that no other processes are executing under the + sandbox uids. This must be done before any chownToBuilder() + calls. */ + killSandbox(false); + /* Right platform? */ if (!parsedDrv->canBuildLocally(worker.store)) throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", @@ -378,35 +477,6 @@ additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); -#if __APPLE__ - if (additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = derivationType.isSandboxed() && !noChroot; - } - - auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir.get()) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - /* Create a temporary directory where the build will take place. */ tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); @@ -482,7 +552,7 @@ temporary build directory. The text files have the format used by `nix-store --register-validity'. However, the deriver fields are left empty. */ - auto s = get(drv->env, "exportReferencesGraph").value_or(""); + auto s = getOr(drv->env, "exportReferencesGraph", ""); Strings ss = tokenizeString(s); if (ss.size() % 2 != 0) throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); @@ -580,12 +650,13 @@ /* Clean up the chroot directory automatically. */ autoDelChroot = std::make_shared(chrootRootDir); - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); + printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootRootDir); - if (mkdir(chrootRootDir.c_str(), 0750) == -1) + // FIXME: make this 0700 + if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) throw SysError("cannot create '%1%'", chrootRootDir); - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) + if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUIDCount() != 1 ? buildUser->getUID() : 0, buildUser->getGID()) == -1) throw SysError("cannot change ownership of '%1%'", chrootRootDir); /* Create a writable /tmp in the chroot. Many builders need @@ -599,6 +670,11 @@ nobody account. The latter is kind of a hack to support Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); + if (parsedDrv->useUidRange()) + chownToBuilder(chrootRootDir + "/etc"); + + if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536)) + throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); /* Declare the build user's group so that programs get a consistent view of the system (e.g., "id -gn"). */ @@ -649,20 +725,35 @@ dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); } -#elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ + if (cgroup) { + if (mkdir(cgroup->c_str(), 0755) != 0) + throw SysError("creating cgroup '%s'", *cgroup); + chownToBuilder(*cgroup); + chownToBuilder(*cgroup + "/cgroup.procs"); + chownToBuilder(*cgroup + "/cgroup.threads"); + //chownToBuilder(*cgroup + "/cgroup.subtree_control"); + } + #else - throw Error("sandboxing builds is not supported on this platform"); + if (parsedDrv->useUidRange()) + throw Error("feature 'uid-range' is not supported on this platform"); + #if __APPLE__ + /* We don't really have any parent prep work to do (yet?) + All work happens in the child, instead. */ + #else + throw Error("sandboxing builds is not supported on this platform"); + #endif #endif + } else { + if (parsedDrv->useUidRange()) + throw Error("feature 'uid-range' is only supported in sandboxed builds"); } if (needsHashRewrite() && pathExists(homeDir)) throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); + printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook); auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : Strings({ worker.store.printStorePath(drvPath) }); enum BuildHookState { @@ -711,15 +802,13 @@ /* Create the log file. */ Path logFile = openLogFile(); - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) + /* Create a pseudoterminal to get the output of the builder. */ + builderOut = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut) throw SysError("opening pseudoterminal master"); // FIXME: not thread-safe, use ptsname_r - std::string slaveName(ptsname(builderOut.readSide.get())); + std::string slaveName = ptsname(builderOut.get()); if (buildUser) { if (chmod(slaveName.c_str(), 0600)) @@ -730,34 +819,34 @@ } #if __APPLE__ else { - if (grantpt(builderOut.readSide.get())) + if (grantpt(builderOut.get())) throw SysError("granting access to pseudoterminal slave"); } #endif - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - - if (unlockpt(builderOut.readSide.get())) + if (unlockpt(builderOut.get())) throw SysError("unlocking pseudoterminal"); - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); + /* Open the slave side of the pseudoterminal and use it as stderr. */ + auto openSlave = [&]() + { + AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); - cfmakeraw(&term); + if (tcsetattr(builderOut.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); + if (dup2(builderOut.get(), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + }; buildResult.startTime = time(0); @@ -804,14 +893,18 @@ userNamespaceSync.create(); - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; + usingUserNamespace = userNamespacesSupported(); - usingUserNamespace = userNamespacesEnabled; + Pipe sendPid; + sendPid.create(); Pid helper = startProcess([&]() { + sendPid.readSide.close(); + + /* We need to open the slave early, before + CLONE_NEWUSER. Otherwise we get EPERM when running as + root. */ + openSlave(); /* Drop additional groups here because we can't do it after we've created the new user namespace. FIXME: @@ -824,51 +917,22 @@ if (getuid() == 0 && setgroups(0, 0) == -1) throw SysError("setgroups failed"); - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + ProcessOptions options; + options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (privateNetwork) - flags |= CLONE_NEWNET; + options.cloneFlags |= CLONE_NEWNET; if (usingUserNamespace) - flags |= CLONE_NEWUSER; + options.cloneFlags |= CLONE_NEWUSER; - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); + pid_t child = startProcess([&]() { runChild(); }, options); - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); + writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); _exit(0); }); - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) + sendPid.writeSide.close(); + + if (helper.wait() != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -879,10 +943,9 @@ userNamespaceSync.writeSide = -1; }); - auto ss = tokenizeString>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - pid = string2Int(ss[1]).value(); + auto ss = tokenizeString>(readLine(sendPid.readSide.get())); + assert(ss.size() == 1); + pid = string2Int(ss[0]).value(); if (usingUserNamespace) { /* Set the UID/GID mapping of the builder's user namespace @@ -890,14 +953,16 @@ the calling user (if build users are disabled). */ uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); + uid_t nrIds = buildUser ? buildUser->getUIDCount() : 1; writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d 1", sandboxUid(), hostUid)); + fmt("%d %d %d", sandboxUid(), hostUid, nrIds)); - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + if (!buildUser || buildUser->getUIDCount() == 1) + writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d 1", sandboxGid(), hostGid)); + fmt("%d %d %d", sandboxGid(), hostGid, nrIds)); } else { debug("note: not using a user namespace"); if (!buildUser) @@ -924,31 +989,32 @@ throw SysError("getting sandbox user namespace"); } + /* Move the child into its own cgroup. */ + if (cgroup) + writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); + /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); } else #endif { -#if __linux__ - fallback: -#endif pid = startProcess([&]() { + openSlave(); runChild(); }); } /* parent */ pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); + worker.childStarted(shared_from_this(), {builderOut.get()}, true, true); /* Check if setting up the build environment failed. */ std::vector msgs; while (true) { std::string msg = [&]() { try { - return readLine(builderOut.readSide.get()); + return readLine(builderOut.get()); } catch (Error & e) { auto status = pid.wait(); e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", @@ -960,7 +1026,7 @@ }(); if (msg.substr(0, 1) == "\2") break; if (msg.substr(0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); + FdSource source(builderOut.get()); auto ex = readError(source); ex.addTrace({}, "while setting up the build environment"); throw ex; @@ -989,7 +1055,7 @@ there is no size constraint). */ if (!parsedDrv->getStructuredAttrs()) { - StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); + StringSet passAsFile = tokenizeString(getOr(drv->env, "passAsFile", "")); for (auto & i : drv->env) { if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; @@ -1044,7 +1110,7 @@ env["NIX_STORE"] = worker.store.storeDir; /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores); initTmpDir(); @@ -1095,10 +1161,10 @@ writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); chownToBuilder(tmpDir + "/.attrs.sh"); - env["NIX_ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh"; + env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh"; writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); chownToBuilder(tmpDir + "/.attrs.json"); - env["NIX_ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json"; + env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json"; } } @@ -1269,7 +1335,7 @@ result.rethrow(); } - std::vector buildPathsWithResults( + std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr) override @@ -1344,17 +1410,20 @@ unknown, downloadSize, narSize); } - virtual std::optional getBuildLog(const StorePath & path) override + virtual std::optional getBuildLogExact(const StorePath & path) override { return std::nullopt; } virtual void addBuildLog(const StorePath & path, std::string_view log) override { unsupported("addBuildLog"); } + + std::optional isTrustedClient() override + { return NotTrusted; } }; void LocalDerivationGoal::startDaemon() { - settings.requireExperimentalFeature(Xp::RecursiveNix); + experimentalFeatureSettings.require(Xp::RecursiveNix); Store::Params params; params["path-info-cache-size"] = "0"; @@ -1401,8 +1470,7 @@ FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); + NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); @@ -1529,6 +1597,22 @@ seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); + if (nativeSystem == "mips64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPS) != 0) + printError("unable to add mips seccomp architecture"); + + if (nativeSystem == "mips64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPS64N32) != 0) + printError("unable to add mips64-*abin32 seccomp architecture"); + + if (nativeSystem == "mips64el-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL) != 0) + printError("unable to add mipsel seccomp architecture"); + + if (nativeSystem == "mips64el-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL64N32) != 0) + printError("unable to add mips64el-*abin32 seccomp architecture"); + /* Prevent builders from creating setuid/setgid binaries. */ for (int perm : { S_ISUID, S_ISGID }) { if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, @@ -1571,9 +1655,11 @@ /* Warning: in the child we should absolutely not make any SQLite calls! */ + bool sendException = true; + try { /* child */ - commonChildInit(builderOut); + commonChildInit(); try { setupSeccomp(); @@ -1690,6 +1776,9 @@ for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" }) if (pathExists(path)) ss.push_back(path); + + if (settings.caFile != "") + dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); } for (auto & i : ss) dirsInChroot.emplace(i, i); @@ -1718,7 +1807,19 @@ for (auto & i : dirsInChroot) { if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); + + #if HAVE_EMBEDDED_SANDBOX_SHELL + if (i.second.source == "__embedded_sandbox_shell__") { + static unsigned char sh[] = { + #include "embedded-sandbox-shell.gen.hh" + }; + auto dst = chrootRootDir + i.first; + createDirs(dirOf(dst)); + writeFile(dst, std::string_view((const char *) sh, sizeof(sh))); + chmod_(dst, 0555); + } else + #endif + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); } /* Bind a new instance of procfs on /proc. */ @@ -1726,6 +1827,13 @@ if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) throw SysError("mounting /proc"); + /* Mount sysfs on /sys. */ + if (buildUser && buildUser->getUIDCount() != 1) { + createDirs(chrootRootDir + "/sys"); + if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) + throw SysError("mounting /sys"); + } + /* Mount a new tmpfs on /dev/shm to ensure that whatever the builder puts in /dev/shm is cleaned up automatically. */ if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, @@ -1755,6 +1863,10 @@ } } + /* Make /etc unwritable */ + if (!parsedDrv->useUidRange()) + chmod_(chrootRootDir + "/etc", 0555); + /* Unshare this mount namespace. This is necessary because pivot_root() below changes the root of the mount namespace. This means that the call to setns() in @@ -1768,6 +1880,12 @@ if (unshare(CLONE_NEWNS) == -1) throw SysError("unsharing mount namespace"); + /* Unshare the cgroup namespace. This means + /proc/self/cgroup will show the child's cgroup as '/' + rather than whatever it is in the parent. */ + if (cgroup && unshare(CLONE_NEWCGROUP) == -1) + throw SysError("unsharing cgroup namespace"); + /* Do the chroot(). */ if (chdir(chrootRootDir.c_str()) == -1) throw SysError("cannot change directory to '%1%'", chrootRootDir); @@ -1805,33 +1923,7 @@ /* Close all other file descriptors. */ closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if ((drv->platform == "i686-linux" - && (settings.thisSystem == "x86_64-linux" - || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) - || drv->platform == "armv7l-linux" - || drv->platform == "armv6l-linux") - { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set 32-bit personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif + setPersonality(drv->platform); /* Disable core dumps by default. */ struct rlimit limit = { 0, RLIM_INFINITY }; @@ -1853,9 +1945,8 @@ if (setUser && buildUser) { /* Preserve supplementary groups of the build user, to allow admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) + auto gids = buildUser->getSupplementaryGIDs(); + if (setgroups(gids.size(), gids.data()) == -1) throw SysError("cannot set supplementary groups of build user"); if (setgid(buildUser->getGID()) == -1 || @@ -1919,10 +2010,14 @@ sandboxProfile += "(deny default (with no-log))\n"; } - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; + sandboxProfile += + #include "sandbox-defaults.sb" + ; if (!derivationType.isSandboxed()) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; + sandboxProfile += + #include "sandbox-network.sb" + ; /* Add the output paths we'll use at build-time to the chroot */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; @@ -1965,7 +2060,9 @@ sandboxProfile += additionalSandboxProfile; } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; + sandboxProfile += + #include "sandbox-minimal.sb" + ; debug("Generated sandbox profile:"); debug(sandboxProfile); @@ -1978,7 +2075,7 @@ /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); + Path globalTmpDir = canonPath(getEnvNonEmpty("TMPDIR").value_or("/tmp"), true); /* They don't like trailing slashes on subpath directives */ if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); @@ -1990,8 +2087,6 @@ args.push_back(sandboxFile); args.push_back("-D"); args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); if (allowLocalNetworking) { args.push_back("-D"); args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1")); @@ -2015,6 +2110,8 @@ /* Indicate that we managed to set up the build environment. */ writeFull(STDERR_FILENO, std::string("\2\n")); + sendException = false; + /* Execute the program. This should not return. */ if (drv->isBuiltin()) { try { @@ -2068,16 +2165,19 @@ throw SysError("executing '%1%'", drv->builder); } catch (Error & e) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); + if (sendException) { + writeFull(STDERR_FILENO, "\1\n"); + FdSink sink(STDERR_FILENO); + sink << e; + sink.flush(); + } else + std::cerr << e.msg(); _exit(1); } } -DrvOutputs LocalDerivationGoal::registerOutputs() +SingleDrvOutputs LocalDerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -2097,7 +2197,6 @@ InodesSeen inodesSeen; Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; std::exception_ptr delayedException; @@ -2128,12 +2227,22 @@ std::map> outputReferencesIfUnregistered; std::map outputStats; for (auto & [outputName, _] : drv->outputs) { - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); + auto scratchOutput = get(scratchOutputs, outputName); + if (!scratchOutput) + throw BuildError( + "builder for '%s' has no scratch output for '%s'", + worker.store.printStorePath(drvPath), outputName); + auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchOutput)); outputsToSort.insert(outputName); /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto & initialInfo = initialOutputs.at(outputName); + auto initialOutput = get(initialOutputs, outputName); + if (!initialOutput) + throw BuildError( + "builder for '%s' has no initial output for '%s'", + worker.store.printStorePath(drvPath), outputName); + auto & initialInfo = *initialOutput; /* Don't register if already valid, and not checking */ initialInfo.wanted = buildMode == bmCheck @@ -2169,13 +2278,33 @@ /* Canonicalise first. This ensures that the path we're rewriting doesn't contain a hard link to /etc/shadow or something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); + canonicalisePathMetaData( + actualPath, + buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, + inodesSeen); - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + bool discardReferences = false; + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { + experimentalFeatureSettings.require(Xp::DiscardReferences); + if (auto output = get(*udr, outputName)) { + if (!output->is_boolean()) + throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); + discardReferences = output->get(); + } + } + } + + StorePathSet references; + if (discardReferences) + debug("discarding references of output '%s'", outputName); + else { + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = scanForReferences(blank, actualPath, referenceablePaths); + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + references = scanForReferences(blank, actualPath, referenceablePaths); + } outputReferencesIfUnregistered.insert_or_assign( outputName, @@ -2185,6 +2314,11 @@ auto sortedOutputNames = topoSort(outputsToSort, {[&](const std::string & name) { + auto orifu = get(outputReferencesIfUnregistered, name); + if (!orifu) + throw BuildError( + "no output reference for '%s' in build of '%s'", + name, worker.store.printStorePath(drvPath)); return std::visit(overloaded { /* Since we'll use the already installed versions of these, we can treat them as leaves and ignore any references they @@ -2199,7 +2333,7 @@ referencedOutputs.insert(o); return referencedOutputs; }, - }, outputReferencesIfUnregistered.at(name)); + }, *orifu); }}, {[&](const std::string & path, const std::string & parent) { // TODO with more -vvvv also show the temporary paths for manual inspection. @@ -2213,9 +2347,10 @@ OutputPathMap finalOutputs; for (auto & outputName : sortedOutputNames) { - auto output = drv->outputs.at(outputName); - auto & scratchPath = scratchOutputs.at(outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); + auto output = get(drv->outputs, outputName); + auto scratchPath = get(scratchOutputs, outputName); + assert(output && scratchPath); + auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchPath)); auto finish = [&](StorePath finalStorePath) { /* Store the final path */ @@ -2223,10 +2358,13 @@ /* The rewrite rule will be used in downstream outputs that refer to use. This is why the topological sort is essential to do first before this for loop. */ - if (scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; + if (*scratchPath != finalStorePath) + outputRewrites[std::string { scratchPath->hashPart() }] = std::string { finalStorePath.hashPart() }; }; + auto orifu = get(outputReferencesIfUnregistered, outputName); + assert(orifu); + std::optional referencesOpt = std::visit(overloaded { [&](const AlreadyRegistered & skippedFinalPath) -> std::optional { finish(skippedFinalPath.path); @@ -2235,7 +2373,7 @@ [&](const PerhapsNeedToRegister & r) -> std::optional { return r.refs; }, - }, outputReferencesIfUnregistered.at(outputName)); + }, *orifu); if (!referencesOpt) continue; @@ -2253,73 +2391,96 @@ sink.s = rewriteStrings(sink.s, outputRewrites); StringSource source(sink.s); restorePath(actualPath, source); + + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, {}, inodesSeen); } }; - auto rewriteRefs = [&]() -> std::pair { + auto rewriteRefs = [&]() -> StoreReferences { /* In the CA case, we need the rewritten refs to calculate the final path, therefore we look for a *non-rewritten self-reference, and use a bool rather try to solve the computationally intractable fixed point. */ - std::pair res { - false, - {}, + StoreReferences res { + .self = false, }; for (auto & r : references) { auto name = r.name(); auto origHash = std::string { r.hashPart() }; - if (r == scratchPath) - res.first = true; - else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); - else { - std::string newRef = outputRewrites.at(origHash); + if (r == *scratchPath) { + res.self = true; + } else if (auto outputRewrite = get(outputRewrites, origHash)) { + std::string newRef = *outputRewrite; newRef += '-'; newRef += name; - res.second.insert(StorePath { newRef }); + res.others.insert(StorePath { newRef }); + } else { + res.others.insert(r); } } return res; }; auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo { - auto & st = outputStats.at(outputName); - if (outputHash.method == FileIngestionMethod::Flat) { + auto st = get(outputStats, outputName); + if (!st) + throw BuildError( + "output path %1% without valid stats info", + actualPath); + if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } || + outputHash.method == ContentAddressMethod { TextIngestionMethod {} }) + { /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) + if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) throw BuildError( "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", + "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", actualPath); } rewriteOutput(); /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath.hashPart() }; + std::string oldHashPart { scratchPath->hashPart() }; HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } + std::visit(overloaded { + [&](const TextIngestionMethod &) { + readFile(actualPath, caSink); + }, + [&](const FileIngestionMethod & m2) { + switch (m2) { + case FileIngestionMethod::Recursive: + dumpPath(actualPath, caSink); + break; + case FileIngestionMethod::Flat: + readFile(actualPath, caSink); + break; + } + }, + }, outputHash.method.raw); auto got = caSink.finish().first; - auto refs = rewriteRefs(); - auto finalPath = worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first); - if (scratchPath != finalPath) { + auto optCA = ContentAddressWithReferences::fromPartsOpt( + outputHash.method, + std::move(got), + rewriteRefs()); + if (!optCA) { + // TODO track distinct failure modes separately (at the time of + // writing there is just one but `nullopt` is unclear) so this + // message can't get out of sync. + throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing"); + } + ValidPathInfo newInfo0 { + worker.store, + outputPathName(drv->name, outputName), + *std::move(optCA), + Hash::dummy, + }; + if (*scratchPath != newInfo0.path) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { - StringSink sink; - dumpPath(actualPath, sink); - RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); - rsink2(sink.s); + RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); + dumpPath(actualPath, rsink2); rsink2.flush(); }); Path tmpPath = actualPath + ".tmp"; @@ -2329,19 +2490,8 @@ } HashResult narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { - finalPath, - narHashAndSize.first, - }; - + newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); assert(newInfo0.ca); return newInfo0; @@ -2354,31 +2504,32 @@ auto requiredFinalPath = output.path; /* Preemptively add rewrite rule for final hash, as that is what the NAR hash will use rather than normalized-self references */ - if (scratchPath != requiredFinalPath) + if (*scratchPath != requiredFinalPath) outputRewrites.insert_or_assign( - std::string { scratchPath.hashPart() }, + std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); rewriteOutput(); auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) + newInfo0.references = std::move(refs.others); + if (refs.self) newInfo0.references.insert(newInfo0.path); return newInfo0; }, [&](const DerivationOutput::CAFixed & dof) { + auto wanted = dof.ca.getHash(); + auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, + .method = dof.ca.getMethod(), + .hashType = wanted.type, }); /* Check wanted hash */ - const Hash & wanted = dof.hash.hash; assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); + auto got = newInfo0.ca->getHash(); if (wanted != got) { /* Throw an error after registering the path as valid. */ @@ -2389,6 +2540,11 @@ wanted.to_string(SRI, true), got.to_string(SRI, true))); } + if (!newInfo0.references.empty()) + delayedException = std::make_exception_ptr( + BuildError("illegal path references in fixed-output derivation '%s'", + worker.store.printStorePath(drvPath))); + return newInfo0; }, @@ -2409,11 +2565,11 @@ }); }, - }, output.raw()); + }, output->raw()); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); + canonicalisePathMetaData(actualPath, {}, inodesSeen); /* Calculate where we'll move the output files. In the checking case we will leave leave them where they are, for now, rather than move to @@ -2425,7 +2581,7 @@ derivations. */ PathLocks dynamicOutputLock; dynamicOutputLock.setDeletion(true); - auto optFixedPath = output.path(worker.store, drv->name, outputName); + auto optFixedPath = output->path(worker.store, drv->name, outputName); if (!optFixedPath || worker.store.printStorePath(*optFixedPath) != finalDestPath) { @@ -2491,17 +2647,14 @@ /* For debugging, print out the referenced and unreferenced paths. */ for (auto & i : inputPaths) { - auto j = references.find(i); - if (j == references.end()) - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); - else + if (references.count(i)) debug("referenced input: '%1%'", worker.store.printStorePath(i)); + else + debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); } - if (curRound == nrRounds) { - localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } + localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() + worker.markContentsGood(newInfo.path); newInfo.deriver = drvPath; newInfo.ultimate = true; @@ -2530,62 +2683,6 @@ /* Apply output checks. */ checkOutputs(infos); - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - buildResult.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - printError(hint); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return {}; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ @@ -2609,24 +2706,25 @@ means it's safe to link the derivation to the output hash. We must do that for floating CA derivations, which otherwise couldn't be cached, but it's fine to do in all cases. */ - DrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs; for (auto & [outputName, newInfo] : infos) { + auto oldinfo = get(initialOutputs, outputName); + assert(oldinfo); auto thisRealisation = Realisation { .id = DrvOutput { - initialOutputs.at(outputName).outputHash, + oldinfo->outputHash, outputName }, .outPath = newInfo.path }; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && drv->type().isPure()) { signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); } - if (wantOutput(outputName, wantedOutputs)) - builtOutputs.emplace(thisRealisation.id, thisRealisation); + builtOutputs.emplace(outputName, thisRealisation); } return builtOutputs; @@ -2710,9 +2808,10 @@ for (auto & i : *value) { if (worker.store.isStorePath(i)) spec.insert(worker.store.parseStorePath(i)); - else if (outputs.count(i)) - spec.insert(outputs.at(i).path); - else throw BuildError("derivation contains an illegal reference specifier '%s'", i); + else if (auto output = get(outputs, i)) + spec.insert(output->path); + else + throw BuildError("derivation contains an illegal reference specifier '%s'", i); } auto used = recursive @@ -2751,24 +2850,18 @@ }; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - auto outputChecks = structuredAttrs->find("outputChecks"); - if (outputChecks != structuredAttrs->end()) { - auto output = outputChecks->find(outputName); - - if (output != outputChecks->end()) { + if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { + if (auto output = get(*outputChecks, outputName)) { Checks checks; - auto maxSize = output->find("maxSize"); - if (maxSize != output->end()) + if (auto maxSize = get(*output, "maxSize")) checks.maxSize = maxSize->get(); - auto maxClosureSize = output->find("maxClosureSize"); - if (maxClosureSize != output->end()) + if (auto maxClosureSize = get(*output, "maxClosureSize")) checks.maxClosureSize = maxClosureSize->get(); - auto get = [&](const std::string & name) -> std::optional { - auto i = output->find(name); - if (i != output->end()) { + auto get_ = [&](const std::string & name) -> std::optional { + if (auto i = get(*output, name)) { Strings res; for (auto j = i->begin(); j != i->end(); ++j) { if (!j->is_string()) @@ -2781,10 +2874,10 @@ return {}; }; - checks.allowedReferences = get("allowedReferences"); - checks.allowedRequisites = get("allowedRequisites"); - checks.disallowedReferences = get("disallowedReferences"); - checks.disallowedRequisites = get("disallowedRequisites"); + checks.allowedReferences = get_("allowedReferences"); + checks.allowedRequisites = get_("allowedRequisites"); + checks.disallowedReferences = get_("disallowedReferences"); + checks.disallowedRequisites = get_("disallowedRequisites"); applyChecks(checks); } @@ -2822,7 +2915,7 @@ bool LocalDerivationGoal::isReadDesc(int fd) { return (hook && DerivationGoal::isReadDesc(fd)) || - (!hook && fd == builderOut.readSide.get()); + (!hook && fd == builderOut.get()); } diff -Nru nix-2.8.0/src/libstore/build/local-derivation-goal.hh nix-2.16.1+dfsg/src/libstore/build/local-derivation-goal.hh --- nix-2.8.0/src/libstore/build/local-derivation-goal.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/local-derivation-goal.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivation-goal.hh" #include "local-store.hh" @@ -9,45 +10,75 @@ { LocalStore & getLocalStore(); - /* User selected for running the builder. */ + /** + * User selected for running the builder. + */ std::unique_ptr buildUser; - /* The process ID of the builder. */ + /** + * The process ID of the builder. + */ Pid pid; - /* The temporary directory. */ + /** + * The cgroup of the builder, if any. + */ + std::optional cgroup; + + /** + * The temporary directory. + */ Path tmpDir; - /* The path of the temporary directory in the sandbox. */ + /** + * The path of the temporary directory in the sandbox. + */ Path tmpDirInSandbox; - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; + /** + * Master side of the pseudoterminal used for the builder's + * standard output/error. + */ + AutoCloseFD builderOut; - /* Pipe for synchronising updates to the builder namespaces. */ + /** + * Pipe for synchronising updates to the builder namespaces. + */ Pipe userNamespaceSync; - /* The mount namespace and user namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ + /** + * The mount namespace and user namespace of the builder, used to add additional + * paths to the sandbox as a result of recursive Nix calls. + */ AutoCloseFD sandboxMountNamespace; AutoCloseFD sandboxUserNamespace; - /* On Linux, whether we're doing the build in its own user - namespace. */ + /** + * On Linux, whether we're doing the build in its own user + * namespace. + */ bool usingUserNamespace = true; - /* Whether we're currently doing a chroot build. */ + /** + * Whether we're currently doing a chroot build. + */ bool useChroot = false; Path chrootRootDir; - /* RAII object to delete the chroot directory. */ + /** + * RAII object to delete the chroot directory. + */ std::shared_ptr autoDelChroot; - /* Whether to run the build in a private network namespace. */ + /** + * Whether to run the build in a private network namespace. + */ bool privateNetwork = false; - /* Stuff we need to pass to initChild(). */ + /** + * Stuff we need to pass to initChild(). + */ struct ChrootPath { Path source; bool optional; @@ -66,56 +97,73 @@ SandboxProfile additionalSandboxProfile; #endif - /* Hash rewriting. */ + /** + * Hash rewriting. + */ StringMap inputRewrites, outputRewrites; typedef map RedirectedOutputs; RedirectedOutputs redirectedOutputs; - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. + /** + * The outputs paths used during the build. + * + * - Input-addressed derivations or fixed content-addressed outputs are + * sometimes built when some of their outputs already exist, and can not + * be hidden via sandboxing. We use temporary locations instead and + * rewrite after the build. Otherwise the regular predetermined paths are + * put here. + * + * - Floating content-addressed derivations do not know their final build + * output paths until the outputs are hashed, so random locations are + * used, and then renamed. The randomness helps guard against hidden + * self-references. */ OutputPathMap scratchOutputs; - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ + /** + * Path registration info from the previous round, if we're + * building multiple times. Since this contains the hash, it + * allows us to compare whether two rounds produced the same + * result. + */ std::map prevInfos; - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } + uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } + gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } const static Path homeDir; - /* The recursive Nix daemon socket. */ + /** + * The recursive Nix daemon socket. + */ AutoCloseFD daemonSocket; - /* The daemon main thread. */ + /** + * The daemon main thread. + */ std::thread daemonThread; - /* The daemon worker threads. */ + /** + * The daemon worker threads. + */ std::vector daemonWorkerThreads; - /* Paths that were added via recursive Nix calls. */ + /** + * Paths that were added via recursive Nix calls. + */ StorePathSet addedPaths; - /* Realisations that were added via recursive Nix calls. */ + /** + * Realisations that were added via recursive Nix calls. + */ std::set addedDrvOutputs; - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ + /** + * Recursive Nix calls are only allowed to build or realize paths + * in the original input closure or added via a recursive Nix call + * (so e.g. you can't do 'nix-store -r /nix/store/' where + * /nix/store/ is some arbitrary path in a binary cache). + */ bool isAllowed(const StorePath & path) { return inputPaths.count(path) || addedPaths.count(path); @@ -133,55 +181,81 @@ virtual ~LocalDerivationGoal() override; - /* Whether we need to perform hash rewriting if there are valid output paths. */ + /** + * Whether we need to perform hash rewriting if there are valid output paths. + */ bool needsHashRewrite(); - /* The additional states. */ + /** + * The additional states. + */ void tryLocalBuild() override; - /* Start building a derivation. */ + /** + * Start building a derivation. + */ void startBuilder(); - /* Fill in the environment for the builder. */ + /** + * Fill in the environment for the builder. + */ void initEnv(); - /* Setup tmp dir location. */ + /** + * Setup tmp dir location. + */ void initTmpDir(); - /* Write a JSON file containing the derivation attributes. */ + /** + * Write a JSON file containing the derivation attributes. + */ void writeStructuredAttrs(); void startDaemon(); void stopDaemon(); - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ + /** + * Add 'path' to the set of paths that may be referenced by the + * outputs, and make it appear in the sandbox. + */ void addDependency(const StorePath & path); - /* Make a file owned by the builder. */ + /** + * Make a file owned by the builder. + */ void chownToBuilder(const Path & path); int getChildStatus() override; - /* Run the builder's process. */ + /** + * Run the builder's process. + */ void runChild(); - /* Check that the derivation outputs all exist and register them - as valid. */ - DrvOutputs registerOutputs() override; + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ + SingleDrvOutputs registerOutputs() override; void signRealisation(Realisation &) override; - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ + /** + * Check that an output meets the requirements specified by the + * 'outputChecks' attribute (or the legacy + * '{allowed,disallowed}{References,Requisites}' attributes). + */ void checkOutputs(const std::map & outputs); - /* Close the read side of the logger pipe. */ + /** + * Close the read side of the logger pipe. + */ void closeReadPipes() override; - /* Cleanup hooks for buildDone() */ + /** + * Cleanup hooks for buildDone() + */ void cleanupHookFinally() override; void cleanupPreChildKill() override; void cleanupPostChildKill() override; @@ -191,20 +265,36 @@ bool isReadDesc(int fd) override; - /* Delete the temporary directory, if we have one. */ + /** + * Delete the temporary directory, if we have one. + */ void deleteTmpDir(bool force); - /* Forcibly kill the child process, if any. */ + /** + * Forcibly kill the child process, if any. + */ void killChild() override; - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ + /** + * Kill any processes running under the build user UID or in the + * cgroup of the build. + */ + void killSandbox(bool getStats); + + /** + * Create alternative path calculated from but distinct from the + * input, so we can avoid overwriting outputs (or other store paths) + * that already exist. + */ StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ + + /** + * Make a path to another based on the output name along with the + * derivation hash. + * + * @todo Add option to randomize, so we can audit whether our + * rewrites caught everything + */ StorePath makeFallbackPath(std::string_view outputName); }; diff -Nru nix-2.8.0/src/libstore/build/personality.cc nix-2.16.1+dfsg/src/libstore/build/personality.cc --- nix-2.8.0/src/libstore/build/personality.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/personality.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,44 @@ +#include "personality.hh" +#include "globals.hh" + +#if __linux__ +#include +#include +#endif + +#include + +namespace nix { + +void setPersonality(std::string_view system) +{ +#if __linux__ + /* Change the personality to 32-bit if we're doing an + i686-linux build on an x86_64-linux machine. */ + struct utsname utsbuf; + uname(&utsbuf); + if ((system == "i686-linux" + && (std::string_view(SYSTEM) == "x86_64-linux" + || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) + || system == "armv7l-linux" + || system == "armv6l-linux") + { + if (personality(PER_LINUX32) == -1) + throw SysError("cannot set 32-bit personality"); + } + + /* Impersonate a Linux 2.6 machine to get some determinism in + builds that depend on the kernel version. */ + if ((system == "i686-linux" || system == "x86_64-linux") && settings.impersonateLinux26) { + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); + } + + /* Disable address space randomization for improved + determinism. */ + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); +#endif +} + +} diff -Nru nix-2.8.0/src/libstore/build/personality.hh nix-2.16.1+dfsg/src/libstore/build/personality.hh --- nix-2.8.0/src/libstore/build/personality.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/personality.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,12 @@ +#pragma once +///@file + +#include + +namespace nix { + +void setPersonality(std::string_view system); + +} + + diff -Nru nix-2.8.0/src/libstore/build/sandbox-defaults.sb nix-2.16.1+dfsg/src/libstore/build/sandbox-defaults.sb --- nix-2.8.0/src/libstore/build/sandbox-defaults.sb 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/sandbox-defaults.sb 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,110 @@ +R""( + +(define TMPDIR (param "_GLOBAL_TMP_DIR")) + +(deny default) + +; Disallow creating setuid/setgid binaries, since that +; would allow breaking build user isolation. +(deny file-write-setugid) + +; Allow forking. +(allow process-fork) + +; Allow reading system information like #CPUs, etc. +(allow sysctl-read) + +; Allow POSIX semaphores and shared memory. +(allow ipc-posix*) + +; Allow socket creation. +(allow system-socket) + +; Allow sending signals within the sandbox. +(allow signal (target same-sandbox)) + +; Allow getpwuid. +(allow mach-lookup (global-name "com.apple.system.opendirectoryd.libinfo")) + +; Access to /tmp. +; The network-outbound/network-inbound ones are for unix domain sockets, which +; we allow access to in TMPDIR (but if we allow them more broadly, you could in +; theory escape the sandbox) +(allow file* process-exec network-outbound network-inbound + (literal "/tmp") (subpath TMPDIR)) + +; Some packages like to read the system version. +(allow file-read* + (literal "/System/Library/CoreServices/SystemVersion.plist") + (literal "/System/Library/CoreServices/SystemVersionCompat.plist")) + +; Without this line clang cannot write to /dev/null, breaking some configure tests. +(allow file-read-metadata (literal "/dev")) + +; Many packages like to do local networking in their test suites, but let's only +; allow it if the package explicitly asks for it. +(if (param "_ALLOW_LOCAL_NETWORKING") + (begin + (allow network* (local ip) (local tcp) (local udp)) + + ; Allow access to /etc/resolv.conf (which is a symlink to + ; /private/var/run/resolv.conf). + ; TODO: deduplicate with sandbox-network.sb + (allow file-read-metadata + (literal "/var") + (literal "/etc") + (literal "/etc/resolv.conf") + (literal "/private/etc/resolv.conf")) + + (allow file-read* + (literal "/private/var/run/resolv.conf")) + + ; Allow DNS lookups. This is even needed for localhost, which lots of tests rely on + (allow file-read-metadata (literal "/etc/hosts")) + (allow file-read* (literal "/private/etc/hosts")) + (allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))))) + +; Standard devices. +(allow file* + (literal "/dev/null") + (literal "/dev/random") + (literal "/dev/stdin") + (literal "/dev/stdout") + (literal "/dev/tty") + (literal "/dev/urandom") + (literal "/dev/zero") + (subpath "/dev/fd")) + +; Allow pseudo-terminals. +(allow file* + (literal "/dev/ptmx") + (regex #"^/dev/pty[a-z]+") + (regex #"^/dev/ttys[0-9]+")) + +; Does nothing, but reduces build noise. +(allow file* (literal "/dev/dtracehelper")) + +; Allow access to zoneinfo since libSystem needs it. +(allow file-read* (subpath "/usr/share/zoneinfo")) + +(allow file-read* (subpath "/usr/share/locale")) + +; This is mostly to get more specific log messages when builds try to +; access something in /etc or /var. +(allow file-read-metadata + (literal "/etc") + (literal "/var") + (literal "/private/var/tmp")) + +; This is used by /bin/sh on macOS 10.15 and later. +(allow file* + (literal "/private/var/select/sh")) + +; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin (and vice versa). +(allow file-read* + (subpath "/Library/Apple/usr/libexec/oah") + (subpath "/System/Library/Apple/usr/libexec/oah") + (subpath "/System/Library/LaunchDaemons/com.apple.oahd.plist") + (subpath "/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) + +)"" diff -Nru nix-2.8.0/src/libstore/build/sandbox-minimal.sb nix-2.16.1+dfsg/src/libstore/build/sandbox-minimal.sb --- nix-2.8.0/src/libstore/build/sandbox-minimal.sb 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/sandbox-minimal.sb 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,9 @@ +R""( + +(allow default) + +; Disallow creating setuid/setgid binaries, since that +; would allow breaking build user isolation. +(deny file-write-setugid) + +)"" diff -Nru nix-2.8.0/src/libstore/build/sandbox-network.sb nix-2.16.1+dfsg/src/libstore/build/sandbox-network.sb --- nix-2.8.0/src/libstore/build/sandbox-network.sb 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/sandbox-network.sb 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,24 @@ +R""( + +; Allow local and remote network traffic. +(allow network* (local ip) (remote ip)) + +; Allow access to /etc/resolv.conf (which is a symlink to +; /private/var/run/resolv.conf). +(allow file-read-metadata + (literal "/var") + (literal "/etc") + (literal "/etc/resolv.conf") + (literal "/private/etc/resolv.conf")) + +(allow file-read* + (literal "/private/var/run/resolv.conf")) + +; Allow DNS lookups. +(allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))) + +; Allow access to trustd. +(allow mach-lookup (global-name "com.apple.trustd")) +(allow mach-lookup (global-name "com.apple.trustd.agent")) + +)"" diff -Nru nix-2.8.0/src/libstore/build/substitution-goal.cc nix-2.16.1+dfsg/src/libstore/build/substitution-goal.cc --- nix-2.8.0/src/libstore/build/substitution-goal.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/substitution-goal.cc 2023-06-06 10:55:23.000000000 +0000 @@ -95,7 +95,9 @@ subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA( + std::string { storePath.name() }, + ContentAddressWithReferences::withoutRefs(*ca)); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { @@ -154,7 +156,7 @@ only after we've downloaded the path. */ if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) { - warn("the substitute for '%s' from '%s' is not signed by any of the keys in 'trusted-public-keys'", + warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", worker.store.printStorePath(storePath), sub->getUri()); tryNext(); return; @@ -198,11 +200,10 @@ { trace("trying to run"); - /* Make sure that we are allowed to start a build. Note that even - if maxBuildJobs == 0 (no local builds allowed), we still allow - a substituter to run. This is because substitutions cannot be - distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { + /* Make sure that we are allowed to start a substitution. Note that even + if maxSubstitutionJobs == 0, we still allow a substituter to run. This + prevents infinite waiting. */ + if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) { worker.waitForBuildSlot(shared_from_this()); return; } diff -Nru nix-2.8.0/src/libstore/build/substitution-goal.hh nix-2.16.1+dfsg/src/libstore/build/substitution-goal.hh --- nix-2.8.0/src/libstore/build/substitution-goal.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/substitution-goal.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "lock.hh" #include "store-api.hh" @@ -10,38 +11,58 @@ struct PathSubstitutionGoal : public Goal { - /* The store path that should be realised through a substitute. */ + /** + * The store path that should be realised through a substitute. + */ StorePath storePath; - /* The path the substituter refers to the path as. This will be - different when the stores have different names. */ + /** + * The path the substituter refers to the path as. This will be + * different when the stores have different names. + */ std::optional subPath; - /* The remaining substituters. */ + /** + * The remaining substituters. + */ std::list> subs; - /* The current substituter. */ + /** + * The current substituter. + */ std::shared_ptr sub; - /* Whether a substituter failed. */ + /** + * Whether a substituter failed. + */ bool substituterFailed = false; - /* Path info returned by the substituter's query info operation. */ + /** + * Path info returned by the substituter's query info operation. + */ std::shared_ptr info; - /* Pipe for the substituter's standard output. */ + /** + * Pipe for the substituter's standard output. + */ Pipe outPipe; - /* The substituter thread. */ + /** + * The substituter thread. + */ std::thread thr; std::promise promise; - /* Whether to try to repair a valid path. */ + /** + * Whether to try to repair a valid path. + */ RepairFlag repair; - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ + /** + * Location where we're downloading the substitute. Differs from + * storePath when doing a repair. + */ Path destPath; std::unique_ptr> maintainExpectedSubstitutions, @@ -50,7 +71,9 @@ typedef void (PathSubstitutionGoal::*GoalState)(); GoalState state; - /* Content address for recomputing store path */ + /** + * Content address for recomputing store path + */ std::optional ca; void done( @@ -64,16 +87,20 @@ void timedOut(Error && ex) override { abort(); }; + /** + * We prepend "a$" to the key name to ensure substitution goals + * happen before derivation goals. + */ std::string key() override { - /* "a$" ensures substitution goals happen before derivation - goals. */ return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } void work() override; - /* The states. */ + /** + * The states. + */ void init(); void tryNext(); void gotInfo(); @@ -81,11 +108,15 @@ void tryToRun(); void finished(); - /* Callback used by the worker to write to the log. */ + /** + * Callback used by the worker to write to the log. + */ void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; void cleanup() override; + + JobCategory jobCategory() override { return JobCategory::Substitution; }; }; } diff -Nru nix-2.8.0/src/libstore/build/worker.cc nix-2.16.1+dfsg/src/libstore/build/worker.cc --- nix-2.8.0/src/libstore/build/worker.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/worker.cc 2023-06-06 10:55:23.000000000 +0000 @@ -18,6 +18,7 @@ { /* Debugging: prevent recursive workers. */ nrLocalBuilds = 0; + nrSubstitutions = 0; lastWokenUp = steady_time_point::min(); permanentFailure = false; timedOut = false; @@ -42,7 +43,7 @@ std::shared_ptr Worker::makeDerivationGoalCommon( const StorePath & drvPath, - const StringSet & wantedOutputs, + const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal) { std::weak_ptr & goal_weak = derivationGoals[drvPath]; @@ -59,7 +60,7 @@ std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode) + const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { return !dynamic_cast(&store) @@ -70,7 +71,7 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) + const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { return !dynamic_cast(&store) @@ -92,6 +93,7 @@ return goal; } + std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional ca) { std::weak_ptr & goal_weak = drvOutputSubstitutionGoals[id]; @@ -104,6 +106,20 @@ return goal; } + +GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & bfd) -> GoalPtr { + return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode); + }, + [&](const DerivedPath::Opaque & bo) -> GoalPtr { + return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); + }, + }, req.raw()); +} + + template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { @@ -161,6 +177,12 @@ } +unsigned Worker::getNrSubstitutions() +{ + return nrSubstitutions; +} + + void Worker::childStarted(GoalPtr goal, const std::set & fds, bool inBuildSlot, bool respectTimeouts) { @@ -172,7 +194,10 @@ child.inBuildSlot = inBuildSlot; child.respectTimeouts = respectTimeouts; children.emplace_back(child); - if (inBuildSlot) nrLocalBuilds++; + if (inBuildSlot) { + if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++; + else nrLocalBuilds++; + } } @@ -183,8 +208,13 @@ if (i == children.end()) return; if (i->inBuildSlot) { - assert(nrLocalBuilds > 0); - nrLocalBuilds--; + if (goal->jobCategory() == JobCategory::Substitution) { + assert(nrSubstitutions > 0); + nrSubstitutions--; + } else { + assert(nrLocalBuilds > 0); + nrLocalBuilds--; + } } children.erase(i); @@ -205,7 +235,9 @@ void Worker::waitForBuildSlot(GoalPtr goal) { debug("wait for build slot"); - if (getNrLocalBuilds() < settings.maxBuildJobs) + bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution; + if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) || + (isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs)) wakeUp(goal); /* we can do it right away */ else addToWeakGoals(wantingToBuild, goal); @@ -276,7 +308,7 @@ if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && 0 == settings.maxBuildJobs) + if (awake.empty() && 0U == settings.maxBuildJobs) { if (getMachines().empty()) throw Error("unable to start any build; either increase '--max-jobs' " @@ -350,7 +382,7 @@ become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ std::vector pollStatus; - std::map fdToPollStatus; + std::map fdToPollStatus; for (auto & i : children) { for (auto & j : i.fds) { pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); @@ -380,7 +412,10 @@ std::set fds2(j->fds); std::vector buffer(4096); for (auto & k : fds2) { - if (pollStatus.at(fdToPollStatus.at(k)).revents) { + const auto fdPollStatusId = get(fdToPollStatus, k); + assert(fdPollStatusId); + assert(*fdPollStatusId < pollStatus.size()); + if (pollStatus.at(*fdPollStatusId).revents) { ssize_t rd = ::read(k, buffer.data(), buffer.size()); // FIXME: is there a cleaner way to handle pt close // than EIO? Is this even standard? diff -Nru nix-2.8.0/src/libstore/build/worker.hh nix-2.16.1+dfsg/src/libstore/build/worker.hh --- nix-2.8.0/src/libstore/build/worker.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build/worker.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "lock.hh" @@ -16,24 +17,29 @@ struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; -/* Workaround for not being able to declare a something like - - class PathSubstitutionGoal : public Goal; - - even when Goal is a complete type. - - This is still a static cast. The purpose of exporting it is to define it in - a place where `PathSubstitutionGoal` is concrete, and use it in a place where it - is opaque. */ +/** + * Workaround for not being able to declare a something like + * + * ```c++ + * class PathSubstitutionGoal : public Goal; + * ``` + * even when Goal is a complete type. + * + * This is still a static cast. The purpose of exporting it is to define it in + * a place where `PathSubstitutionGoal` is concrete, and use it in a place where it + * is opaque. + */ GoalPtr upcast_goal(std::shared_ptr subGoal); GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ +/** + * A mapping used to remember for each child process to what goal it + * belongs, and file descriptors for receiving log data and output + * path creation commands. + */ struct Child { WeakGoalPtr goal; @@ -41,14 +47,19 @@ std::set fds; bool respectTimeouts; bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ + /** + * Time we last got output on stdout/stderr + */ + steady_time_point lastOutput; steady_time_point timeStarted; }; /* Forward definition. */ struct HookInstance; -/* The worker class. */ +/** + * The worker class. + */ class Worker { private: @@ -56,38 +67,63 @@ /* Note: the worker should only have strong pointers to the top-level goals. */ - /* The top-level goals of the worker. */ + /** + * The top-level goals of the worker. + */ Goals topGoals; - /* Goals that are ready to do some work. */ + /** + * Goals that are ready to do some work. + */ WeakGoals awake; - /* Goals waiting for a build slot. */ + /** + * Goals waiting for a build slot. + */ WeakGoals wantingToBuild; - /* Child processes currently running. */ + /** + * Child processes currently running. + */ std::list children; - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ + /** + * Number of build slots occupied. This includes local builds but does not + * include substitutions or remote builds via the build hook. + */ unsigned int nrLocalBuilds; - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ + /** + * Number of substitution slots occupied. + */ + unsigned int nrSubstitutions; + + /** + * Maps used to prevent multiple instantiations of a goal for the + * same derivation / path. + */ std::map> derivationGoals; std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; - /* Goals waiting for busy paths to be unlocked. */ + /** + * Goals waiting for busy paths to be unlocked. + */ WeakGoals waitingForAnyGoal; - /* Goals sleeping for a few seconds (polling a lock). */ + /** + * Goals sleeping for a few seconds (polling a lock). + */ WeakGoals waitingForAWhile; - /* Last time the goals in `waitingForAWhile' where woken up. */ + /** + * Last time the goals in `waitingForAWhile` where woken up. + */ steady_time_point lastWokenUp; - /* Cache for pathContentsGood(). */ + /** + * Cache for pathContentsGood(). + */ std::map pathContentsGoodCache; public: @@ -96,17 +132,25 @@ const Activity actDerivations; const Activity actSubstitutions; - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ + /** + * Set if at least one derivation had a BuildError (i.e. permanent + * failure). + */ bool permanentFailure; - /* Set if at least one derivation had a timeout. */ + /** + * Set if at least one derivation had a timeout. + */ bool timedOut; - /* Set if at least one derivation fails with a hash mismatch. */ + /** + * Set if at least one derivation fails with a hash mismatch. + */ bool hashMismatch; - /* Set if at least one derivation is not deterministic in check mode. */ + /** + * Set if at least one derivation is not deterministic in check mode. + */ bool checkMismatch; Store & store; @@ -128,78 +172,120 @@ uint64_t expectedNarSize = 0; uint64_t doneNarSize = 0; - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ + /** + * Whether to ask the build hook if it can build a derivation. If + * it answers with "decline-permanently", we don't try again. + */ bool tryBuildHook = true; Worker(Store & store, Store & evalStore); ~Worker(); - /* Make a goal (with caching). */ - - /* derivation goal */ + /** + * Make a goal (with caching). + */ + + /** + * @ref DerivationGoal "derivation goal" + */ private: std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, + const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal); public: std::shared_ptr makeDerivationGoal( const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); - /* substitution goal */ + /** + * @ref SubstitutionGoal "substitution goal" + */ std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - /* Remove a dead goal. */ + /** + * Make a goal corresponding to the `DerivedPath`. + * + * It will be a `DerivationGoal` for a `DerivedPath::Built` or + * a `SubstitutionGoal` for a `DerivedPath::Opaque`. + */ + GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal); + + /** + * Remove a dead goal. + */ void removeGoal(GoalPtr goal); - /* Wake up a goal (i.e., there is something for it to do). */ + /** + * Wake up a goal (i.e., there is something for it to do). + */ void wakeUp(GoalPtr goal); - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ + /** + * Return the number of local build processes currently running (but not + * remote builds via the build hook). + */ unsigned int getNrLocalBuilds(); - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ + /** + * Return the number of substitution processes currently running. + */ + unsigned int getNrSubstitutions(); + + /** + * Registers a running child process. `inBuildSlot` means that + * the process counts towards the jobs limit. + */ void childStarted(GoalPtr goal, const std::set & fds, bool inBuildSlot, bool respectTimeouts); - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ + /** + * Unregisters a running child process. `wakeSleepers` should be + * false if there is no sense in waking up goals that are sleeping + * because they can't run yet (e.g., there is no free build slot, + * or the hook would still say `postpone`). + */ void childTerminated(Goal * goal, bool wakeSleepers = true); - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ + /** + * Put `goal` to sleep until a build slot becomes available (which + * might be right away). + */ void waitForBuildSlot(GoalPtr goal); - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ + /** + * Wait for any goal to finish. Pretty indiscriminate way to + * wait for some resource that some other goal is holding. + */ void waitForAnyGoal(GoalPtr goal); - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ + /** + * Wait for a few seconds and then retry this goal. Used when + * waiting for a lock held by another process. This kind of + * polling is inefficient, but POSIX doesn't really provide a way + * to wait for multiple locks in the main select() loop. + */ void waitForAWhile(GoalPtr goal); - /* Loop until the specified top-level goals have finished. */ + /** + * Loop until the specified top-level goals have finished. + */ void run(const Goals & topGoals); - /* Wait for input to become available. */ + /** + * Wait for input to become available. + */ void waitForInput(); unsigned int exitStatus(); - /* Check whether the given valid path exists and has the right - contents. */ + /** + * Check whether the given valid path exists and has the right + * contents. + */ bool pathContentsGood(const StorePath & path); void markContentsGood(const StorePath & path); diff -Nru nix-2.8.0/src/libstore/build-result.hh nix-2.16.1+dfsg/src/libstore/build-result.hh --- nix-2.8.0/src/libstore/build-result.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/build-result.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,19 +1,23 @@ #pragma once +///@file #include "realisation.hh" #include "derived-path.hh" #include #include - +#include namespace nix { struct BuildResult { - /* Note: don't remove status codes, and only add new status codes - at the end of the list, to prevent client/server - incompatibilities in the nix-store --serve protocol. */ + /** + * @note This is directly used in the nix-store --serve protocol. + * That means we need to worry about compatability across versions. + * Therefore, don't remove status codes, and only add new status + * codes at the end of the list. + */ enum Status { Built = 0, Substituted, @@ -21,8 +25,10 @@ PermanentFailure, InputRejected, OutputRejected, - TransientFailure, // possibly transient - CachedFailure, // no longer used + /// possibly transient + TransientFailure, + /// no longer used + CachedFailure, TimedOut, MiscFailure, DependencyFailed, @@ -32,7 +38,12 @@ NoSubstituters, } status = MiscFailure; - // FIXME: include entire ErrorInfo object. + /** + * Information about the error if the build failed. + * + * @todo This should be an entire ErrorInfo object, not just a + * string, for richer information. + */ std::string errorMsg; std::string toString() const { @@ -52,32 +63,43 @@ case LogLimitExceeded: return "LogLimitExceeded"; case NotDeterministic: return "NotDeterministic"; case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid"; + case NoSubstituters: return "NoSubstituters"; default: return "Unknown"; }; }(); return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); } - /* How many times this build was performed. */ + /** + * How many times this build was performed. + */ unsigned int timesBuilt = 0; - /* If timesBuilt > 1, whether some builds did not produce the same - result. (Note that 'isNonDeterministic = false' does not mean - the build is deterministic, just that we don't have evidence of - non-determinism.) */ + /** + * If timesBuilt > 1, whether some builds did not produce the same + * result. (Note that 'isNonDeterministic = false' does not mean + * the build is deterministic, just that we don't have evidence of + * non-determinism.) + */ bool isNonDeterministic = false; - /* The derivation we built or the store path we substituted. */ - DerivedPath path; - - /* For derivations, a mapping from the names of the wanted outputs - to actual paths. */ - DrvOutputs builtOutputs; - - /* The start/stop times of the build (or one of the rounds, if it - was repeated). */ + /** + * For derivations, a mapping from the names of the wanted outputs + * to actual paths. + */ + SingleDrvOutputs builtOutputs; + + /** + * The start/stop times of the build (or one of the rounds, if it + * was repeated). + */ time_t startTime = 0, stopTime = 0; + /** + * User and system CPU time the build took. + */ + std::optional cpuUser, cpuSystem; + bool success() { return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; @@ -89,4 +111,15 @@ } }; +/** + * A `BuildResult` together with its "primary key". + */ +struct KeyedBuildResult : BuildResult +{ + /** + * The derivation we built or the store path we substituted. + */ + DerivedPath path; +}; + } diff -Nru nix-2.8.0/src/libstore/builtins/buildenv.cc nix-2.16.1+dfsg/src/libstore/builtins/buildenv.cc --- nix-2.8.0/src/libstore/builtins/buildenv.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/builtins/buildenv.cc 2023-06-06 10:55:23.000000000 +0000 @@ -92,12 +92,11 @@ if (S_ISLNK(dstSt.st_mode)) { auto prevPriority = state.priorities[dstFile]; if (prevPriority == priority) - throw Error( - "packages '%1%' and '%2%' have the same priority %3%; " - "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " - "to change the priority of one of the conflicting packages" - " (0 being the highest priority)", - srcFile, readLink(dstFile), priority); + throw BuildEnvFileConflictError( + readLink(dstFile), + srcFile, + priority + ); if (prevPriority < priority) continue; if (unlink(dstFile.c_str()) == -1) diff -Nru nix-2.8.0/src/libstore/builtins/buildenv.hh nix-2.16.1+dfsg/src/libstore/builtins/buildenv.hh --- nix-2.8.0/src/libstore/builtins/buildenv.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/builtins/buildenv.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" #include "store-api.hh" @@ -12,6 +13,32 @@ Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {} }; +class BuildEnvFileConflictError : public Error +{ +public: + const Path fileA; + const Path fileB; + int priority; + + BuildEnvFileConflictError( + const Path fileA, + const Path fileB, + int priority + ) + : Error( + "Unable to build profile. There is a conflict for the following files:\n" + "\n" + " %1%\n" + " %2%", + fileA, + fileB + ) + , fileA(fileA) + , fileB(fileB) + , priority(priority) + {} +}; + typedef std::vector Packages; void buildProfile(const Path & out, Packages && pkgs); diff -Nru nix-2.8.0/src/libstore/builtins/fetchurl.cc nix-2.16.1+dfsg/src/libstore/builtins/fetchurl.cc --- nix-2.8.0/src/libstore/builtins/fetchurl.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/builtins/fetchurl.cc 2023-06-06 10:55:23.000000000 +0000 @@ -24,7 +24,7 @@ Path storePath = getAttr("out"); auto mainUrl = getAttr("url"); - bool unpack = get(drv.env, "unpack").value_or("") == "1"; + bool unpack = getOr(drv.env, "unpack", "") == "1"; /* Note: have to use a fresh fileTransfer here because we're in a forked process. */ diff -Nru nix-2.8.0/src/libstore/builtins/unpack-channel.cc nix-2.16.1+dfsg/src/libstore/builtins/unpack-channel.cc --- nix-2.8.0/src/libstore/builtins/unpack-channel.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/builtins/unpack-channel.cc 2023-06-06 10:55:23.000000000 +0000 @@ -22,8 +22,7 @@ auto entries = readDirectory(out); if (entries.size() != 1) throw Error("channel tarball '%s' contains more than one file", src); - if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1) - throw SysError("renaming channel directory"); + renameFile((out + "/" + entries[0].name), (out + "/" + channelName)); } } diff -Nru nix-2.8.0/src/libstore/builtins.hh nix-2.16.1+dfsg/src/libstore/builtins.hh --- nix-2.8.0/src/libstore/builtins.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/builtins.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" diff -Nru nix-2.8.0/src/libstore/ca-specific-schema.sql nix-2.16.1+dfsg/src/libstore/ca-specific-schema.sql --- nix-2.8.0/src/libstore/ca-specific-schema.sql 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/ca-specific-schema.sql 2023-06-06 10:55:23.000000000 +0000 @@ -13,12 +13,27 @@ create index if not exists IndexRealisations on Realisations(drvPath, outputName); +-- We can end-up in a weird edge-case where a path depends on itself because +-- it’s an output of a CA derivation, that happens to be the same as one of its +-- dependencies. +-- In that case we have a dependency loop (path -> realisation1 -> realisation2 +-- -> path) that we need to break by removing the dependencies between the +-- realisations +create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths + begin + delete from RealisationsRefs where realisationReference in ( + select id from Realisations where outputPath = old.id + ); + end; + create table if not exists RealisationsRefs ( referrer integer not null, realisationReference integer, foreign key (referrer) references Realisations(id) on delete cascade, foreign key (realisationReference) references Realisations(id) on delete restrict ); +-- used by deletion trigger +create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); -- used by QueryRealisationReferences create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); diff -Nru nix-2.8.0/src/libstore/content-address.cc nix-2.16.1+dfsg/src/libstore/content-address.cc --- nix-2.8.0/src/libstore/content-address.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/content-address.cc 2023-06-06 10:55:23.000000000 +0000 @@ -9,7 +9,7 @@ return makeFileIngestionPrefix(method) + printHashType(hash.type); } -std::string makeFileIngestionPrefix(const FileIngestionMethod m) +std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: @@ -21,41 +21,58 @@ } } -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) +std::string ContentAddressMethod::renderPrefix() const { - return "fixed:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base32, true); + return std::visit(overloaded { + [](TextIngestionMethod) -> std::string { return "text:"; }, + [](FileIngestionMethod m2) { + /* Not prefixed for back compat with things that couldn't produce text before. */ + return makeFileIngestionPrefix(m2); + }, + }, raw); +} + +ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) +{ + ContentAddressMethod method = FileIngestionMethod::Flat; + if (splitPrefix(m, "r:")) + method = FileIngestionMethod::Recursive; + else if (splitPrefix(m, "text:")) + method = TextIngestionMethod {}; + return method; } -std::string renderContentAddress(ContentAddress ca) +std::string ContentAddress::render() const { return std::visit(overloaded { - [](TextHash & th) { - return "text:" + th.hash.to_string(Base32, true); + [](const TextHash & th) { + return "text:" + + th.hash.to_string(Base32, true); }, - [](FixedOutputHash & fsh) { - return makeFixedOutputCA(fsh.method, fsh.hash); + [](const FixedOutputHash & fsh) { + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } - }, ca); + }, raw); } -std::string renderContentAddressMethod(ContentAddressMethod cam) +std::string ContentAddressMethod::render(HashType ht) const { return std::visit(overloaded { - [](TextHashMethod & th) { - return std::string{"text:"} + printHashType(htSHA256); + [&](const TextIngestionMethod & th) { + return std::string{"text:"} + printHashType(ht); }, - [](FixedOutputHashMethod & fshm) { - return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); + [&](const FileIngestionMethod & fim) { + return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht); } - }, cam); + }, raw); } -/* - Parses content address strings up to the hash. +/** + * Parses content address strings up to the hash. */ -static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) +static std::pair parseContentAddressMethodPrefix(std::string_view & rest) { std::string_view wholeInput { rest }; @@ -79,46 +96,47 @@ if (prefix == "text") { // No parsing of the ingestion method, "text" only support flat. HashType hashType = parseHashType_(); - if (hashType != htSHA256) - throw Error("text content address hash should use %s, but instead uses %s", - printHashType(htSHA256), printHashType(hashType)); - return TextHashMethod {}; + return { + TextIngestionMethod {}, + std::move(hashType), + }; } else if (prefix == "fixed") { // Parse method auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; HashType hashType = parseHashType_(); - return FixedOutputHashMethod { - .fileIngestionMethod = method, - .hashType = std::move(hashType), + return { + std::move(method), + std::move(hashType), }; } else throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); } -ContentAddress parseContentAddress(std::string_view rawCa) { +ContentAddress ContentAddress::parse(std::string_view rawCa) +{ auto rest = rawCa; - ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); + auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest); + auto hashType = hashType_; // work around clang bug - return std::visit( - overloaded { - [&](TextHashMethod & thm) { - return ContentAddress(TextHash { - .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256) - }); - }, - [&](FixedOutputHashMethod & fohMethod) { - return ContentAddress(FixedOutputHash { - .method = fohMethod.fileIngestionMethod, - .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), - }); - }, - }, caMethod); + return std::visit(overloaded { + [&](TextIngestionMethod &) { + return ContentAddress(TextHash { + .hash = Hash::parseNonSRIUnprefixed(rest, hashType) + }); + }, + [&](FileIngestionMethod & fim) { + return ContentAddress(FixedOutputHash { + .method = fim, + .hash = Hash::parseNonSRIUnprefixed(rest, hashType), + }); + }, + }, caMethod.raw); } -ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) +std::pair ContentAddressMethod::parse(std::string_view caMethod) { std::string asPrefix = std::string{caMethod} + ":"; // parseContentAddressMethodPrefix takes its argument by reference @@ -126,26 +144,144 @@ return parseContentAddressMethodPrefix(asPrefixView); } -std::optional parseContentAddressOpt(std::string_view rawCaOpt) +std::optional ContentAddress::parseOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional() : parseContentAddress(rawCaOpt); + return rawCaOpt == "" + ? std::nullopt + : std::optional { ContentAddress::parse(rawCaOpt) }; }; std::string renderContentAddress(std::optional ca) { - return ca ? renderContentAddress(*ca) : ""; + return ca ? ca->render() : ""; } -Hash getContentAddressHash(const ContentAddress & ca) +ContentAddress ContentAddress::fromParts( + ContentAddressMethod method, Hash hash) noexcept { return std::visit(overloaded { - [](const TextHash & th) { + [&](TextIngestionMethod _) -> ContentAddress { + return TextHash { + .hash = std::move(hash), + }; + }, + [&](FileIngestionMethod m2) -> ContentAddress { + return FixedOutputHash { + .method = std::move(m2), + .hash = std::move(hash), + }; + }, + }, method.raw); +} + +ContentAddressMethod ContentAddress::getMethod() const +{ + return std::visit(overloaded { + [](const TextHash & th) -> ContentAddressMethod { + return TextIngestionMethod {}; + }, + [](const FixedOutputHash & fsh) -> ContentAddressMethod { + return fsh.method; + }, + }, raw); +} + +const Hash & ContentAddress::getHash() const +{ + return std::visit(overloaded { + [](const TextHash & th) -> auto & { return th.hash; }, - [](const FixedOutputHash & fsh) { + [](const FixedOutputHash & fsh) -> auto & { return fsh.hash; - } - }, ca); + }, + }, raw); +} + +std::string ContentAddress::printMethodAlgo() const +{ + return getMethod().renderPrefix() + + printHashType(getHash().type); +} + +bool StoreReferences::empty() const +{ + return !self && others.empty(); +} + +size_t StoreReferences::size() const +{ + return (self ? 1 : 0) + others.size(); +} + +ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept +{ + return std::visit(overloaded { + [&](const TextHash & h) -> ContentAddressWithReferences { + return TextInfo { + .hash = h, + .references = {}, + }; + }, + [&](const FixedOutputHash & h) -> ContentAddressWithReferences { + return FixedOutputInfo { + .hash = h, + .references = {}, + }; + }, + }, ca.raw); +} + +std::optional ContentAddressWithReferences::fromPartsOpt( + ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept +{ + return std::visit(overloaded { + [&](TextIngestionMethod _) -> std::optional { + if (refs.self) + return std::nullopt; + return ContentAddressWithReferences { + TextInfo { + .hash = { .hash = std::move(hash) }, + .references = std::move(refs.others), + } + }; + }, + [&](FileIngestionMethod m2) -> std::optional { + return ContentAddressWithReferences { + FixedOutputInfo { + .hash = { + .method = m2, + .hash = std::move(hash), + }, + .references = std::move(refs), + } + }; + }, + }, method.raw); +} + +ContentAddressMethod ContentAddressWithReferences::getMethod() const +{ + return std::visit(overloaded { + [](const TextInfo & th) -> ContentAddressMethod { + return TextIngestionMethod {}; + }, + [](const FixedOutputInfo & fsh) -> ContentAddressMethod { + return fsh.hash.method; + }, + }, raw); +} + +Hash ContentAddressWithReferences::getHash() const +{ + return std::visit(overloaded { + [](const TextInfo & th) { + return th.hash.hash; + }, + [](const FixedOutputInfo & fsh) { + return fsh.hash.hash; + }, + }, raw); } } diff -Nru nix-2.8.0/src/libstore/content-address.hh nix-2.16.1+dfsg/src/libstore/content-address.hh --- nix-2.8.0/src/libstore/content-address.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/content-address.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,77 +1,317 @@ #pragma once +///@file #include #include "hash.hh" +#include "path.hh" +#include "comparator.hh" namespace nix { +/* + * Content addressing method + */ + +/* We only have one way to hash text with references, so this is a single-value + type, mainly useful with std::variant. +*/ + +/** + * The single way we can serialize "text" file system objects. + * + * Somewhat obscure, used by \ref Derivation derivations and + * `builtins.toFile` currently. + * + * TextIngestionMethod is identical to FileIngestionMethod::Fixed except that + * the former may not have self-references and is tagged `text:${algo}:${hash}` + * rather than `fixed:${algo}:${hash}`. The contents of the store path are + * ingested and hashed identically, aside from the slightly different tag and + * restriction on self-references. + */ +struct TextIngestionMethod : std::monostate { }; + +/** + * An enumeration of the main ways we can serialize file system + * objects. + */ enum struct FileIngestionMethod : uint8_t { + /** + * Flat-file hashing. Directly ingest the contents of a single file + */ Flat = false, + /** + * Recursive (or NAR) hashing. Serializes the file-system object in Nix + * Archive format and ingest that + */ Recursive = true }; +/** + * Compute the prefix to the hash algorithm which indicates how the + * files were ingested. + */ +std::string makeFileIngestionPrefix(FileIngestionMethod m); + +/** + * An enumeration of all the ways we can serialize file system objects. + * + * Just the type of a content address. Combine with the hash itself, and + * we have a `ContentAddress` as defined below. Combine that, in turn, + * with info on references, and we have `ContentAddressWithReferences`, + * as defined further below. + */ +struct ContentAddressMethod +{ + typedef std::variant< + TextIngestionMethod, + FileIngestionMethod + > Raw; + + Raw raw; + + GENERATE_CMP(ContentAddressMethod, me->raw); + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddressMethod(auto &&... arg) + : raw(std::forward(arg)...) + { } + + + /** + * Parse the prefix tag which indicates how the files + * were ingested, with the fixed output case not prefixed for back + * compat. + * + * @param [in] m A string that should begin with the prefix. + * @param [out] m The remainder of the string after the prefix. + */ + static ContentAddressMethod parsePrefix(std::string_view & m); + + /** + * Render the prefix tag which indicates how the files wre ingested. + * + * The rough inverse of `parsePrefix()`. + */ + std::string renderPrefix() const; + + /** + * Parse a content addressing method and hash type. + */ + static std::pair parse(std::string_view rawCaMethod); + + /** + * Render a content addressing method and hash type in a + * nicer way, prefixing both cases. + * + * The rough inverse of `parse()`. + */ + std::string render(HashType ht) const; +}; + + +/* + * Mini content address + */ + +/** + * Somewhat obscure, used by \ref Derivation derivations and + * `builtins.toFile` currently. + */ struct TextHash { + /** + * Hash of the contents of the text/file. + */ Hash hash; + + GENERATE_CMP(TextHash, me->hash); }; -/// Pair of a hash, and how the file system was ingested +/** + * Used by most store objects that are content-addressed. + */ struct FixedOutputHash { + /** + * How the file system objects are serialized + */ FileIngestionMethod method; + /** + * Hash of that serialization + */ Hash hash; + std::string printMethodAlgo() const; + + GENERATE_CMP(FixedOutputHash, me->method, me->hash); }; -/* - We've accumulated several types of content-addressed paths over the years; - fixed-output derivations support multiple hash algorithms and serialisation - methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms: +/** + * We've accumulated several types of content-addressed paths over the + * years; fixed-output derivations support multiple hash algorithms and + * serialisation methods (flat file vs NAR). Thus, ‘ca’ has one of the + * following forms: + * + * - ‘text:sha256:’: For paths + * computed by Store::makeTextPath() / Store::addTextToStore(). + * + * - ‘fixed:::’: For paths computed by + * Store::makeFixedOutputPath() / Store::addToStore(). + */ +struct ContentAddress +{ + typedef std::variant< + TextHash, + FixedOutputHash + > Raw; + + Raw raw; + + GENERATE_CMP(ContentAddress, me->raw); + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddress(auto &&... arg) + : raw(std::forward(arg)...) + { } + + /** + * Compute the content-addressability assertion + * (`ValidPathInfo::ca`) for paths created by + * `Store::makeFixedOutputPath()` / `Store::addToStore()`. + */ + std::string render() const; + + static ContentAddress parse(std::string_view rawCa); + + static std::optional parseOpt(std::string_view rawCaOpt); + + /** + * Create a `ContentAddress` from 2 parts: + * + * @param method Way ingesting the file system data. + * + * @param hash Hash of ingested file system data. + */ + static ContentAddress fromParts( + ContentAddressMethod method, Hash hash) noexcept; - * ‘text:sha256:’: For paths - computed by makeTextPath() / addTextToStore(). + ContentAddressMethod getMethod() const; - * ‘fixed:::’: For paths computed by - makeFixedOutputPath() / addToStore(). -*/ -typedef std::variant< - TextHash, // for paths computed by makeTextPath() / addTextToStore - FixedOutputHash // for path computed by makeFixedOutputPath -> ContentAddress; - -/* Compute the prefix to the hash algorithm which indicates how the files were - ingested. */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); - -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); + const Hash & getHash() const; -std::string renderContentAddress(ContentAddress ca); + std::string printMethodAlgo() const; +}; +/** + * Render the `ContentAddress` if it exists to a string, return empty + * string otherwise. + */ std::string renderContentAddress(std::optional ca); -ContentAddress parseContentAddress(std::string_view rawCa); -std::optional parseContentAddressOpt(std::string_view rawCaOpt); +/* + * Full content address + * + * See the schema for store paths in store-api.cc + */ + +/** + * A set of references to other store objects. + * + * References to other store objects are tracked with store paths, self + * references however are tracked with a boolean. + */ +struct StoreReferences { + /** + * References to other store objects + */ + StorePathSet others; + + /** + * Reference to this store object + */ + bool self = false; + + /** + * @return true iff no references, i.e. others is empty and self is + * false. + */ + bool empty() const; + + /** + * Returns the numbers of references, i.e. the size of others + 1 + * iff self is true. + */ + size_t size() const; -Hash getContentAddressHash(const ContentAddress & ca); + GENERATE_CMP(StoreReferences, me->self, me->others); +}; -/* - We only have one way to hash text with references, so this is single-value - type is only useful in std::variant. -*/ -struct TextHashMethod { }; -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; +// This matches the additional info that we need for makeTextPath +struct TextInfo { + TextHash hash; + /** + * References to other store objects only; self references + * disallowed + */ + StorePathSet references; + + GENERATE_CMP(TextInfo, me->hash, me->references); +}; + +struct FixedOutputInfo { + FixedOutputHash hash; + /** + * References to other store objects or this one. + */ + StoreReferences references; + + GENERATE_CMP(FixedOutputInfo, me->hash, me->references); }; -typedef std::variant< - TextHashMethod, - FixedOutputHashMethod - > ContentAddressMethod; +/** + * Ways of content addressing but not a complete ContentAddress. + * + * A ContentAddress without a Hash. + */ +struct ContentAddressWithReferences +{ + typedef std::variant< + TextInfo, + FixedOutputInfo + > Raw; + + Raw raw; + + GENERATE_CMP(ContentAddressWithReferences, me->raw); + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddressWithReferences(auto &&... arg) + : raw(std::forward(arg)...) + { } + + /** + * Create a `ContentAddressWithReferences` from a mere + * `ContentAddress`, by claiming no references. + */ + static ContentAddressWithReferences withoutRefs(const ContentAddress &) noexcept; + + /** + * Create a `ContentAddressWithReferences` from 3 parts: + * + * @param method Way ingesting the file system data. + * + * @param hash Hash of ingested file system data. + * + * @param refs References to other store objects or oneself. + * + * Do note that not all combinations are supported; `nullopt` is + * returns for invalid combinations. + */ + static std::optional fromPartsOpt( + ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept; -ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + ContentAddressMethod getMethod() const; -std::string renderContentAddressMethod(ContentAddressMethod caMethod); + Hash getHash() const; +}; } diff -Nru nix-2.8.0/src/libstore/crypto.hh nix-2.16.1+dfsg/src/libstore/crypto.hh --- nix-2.8.0/src/libstore/crypto.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/crypto.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" @@ -11,8 +12,10 @@ std::string name; std::string key; - /* Construct Key from a string in the format - ‘:’. */ + /** + * Construct Key from a string in the format + * ‘:’. + */ Key(std::string_view s); std::string to_string() const; @@ -28,7 +31,9 @@ { SecretKey(std::string_view s); - /* Return a detached signature of the given string. */ + /** + * Return a detached signature of the given string. + */ std::string signDetached(std::string_view s) const; PublicKey toPublicKey() const; @@ -52,8 +57,10 @@ typedef std::map PublicKeys; -/* Return true iff ‘sig’ is a correct signature over ‘data’ using one - of the given public keys. */ +/** + * @return true iff ‘sig’ is a correct signature over ‘data’ using one + * of the given public keys. + */ bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys); diff -Nru nix-2.8.0/src/libstore/daemon.cc nix-2.16.1+dfsg/src/libstore/daemon.cc --- nix-2.8.0/src/libstore/daemon.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/daemon.cc 2023-06-06 10:55:23.000000000 +0000 @@ -67,12 +67,12 @@ state->pendingMsgs.push_back(s); } - void log(Verbosity lvl, const FormatOrString & fs) override + void log(Verbosity lvl, std::string_view s) override { if (lvl > verbosity) return; StringSink buf; - buf << STDERR_NEXT << (fs.s + "\n"); + buf << STDERR_NEXT << (s + "\n"); enqueueMsg(buf.s); } @@ -222,7 +222,8 @@ else if (!hasSuffix(s, "/") && trusted.count(s + "/")) subs.push_back(s + "/"); else - warn("ignoring untrusted substituter '%s'", s); + warn("ignoring untrusted substituter '%s', you are not a trusted user.\n" + "Run `man nix.conf` for more information on the `substituters` configuration option.", s); res = subs; return true; }; @@ -230,15 +231,20 @@ try { if (name == "ssh-auth-sock") // obsolete ; - else if (name == settings.experimentalFeatures.name) { + else if (name == experimentalFeatureSettings.experimentalFeatures.name) { // We don’t want to forward the experimental features to // the daemon, as that could cause some pretty weird stuff - if (parseFeatures(tokenizeString(value)) != settings.experimentalFeatures.get()) + if (parseFeatures(tokenizeString(value)) != experimentalFeatureSettings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); + } else if (name == settings.pluginFiles.name) { + if (tokenizeString(value) != settings.pluginFiles.get()) + warn("Ignoring the client-specified plugin-files.\n" + "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); } else if (trusted || name == settings.buildTimeout.name - || name == settings.buildRepeat.name + || name == settings.maxSilentTime.name + || name == settings.pollInterval.name || name == "connect-timeout" || (name == "builders" && value == "")) settings.set(name, value); @@ -257,7 +263,7 @@ { std::vector reqs; if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { - reqs = worker_proto::read(store, from, Phantom> {}); + reqs = WorkerProto>::read(store, from); } else { for (auto & s : readStrings(from)) reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); @@ -281,7 +287,7 @@ } case wopQueryValidPaths: { - auto paths = worker_proto::read(*store, from, Phantom {}); + auto paths = WorkerProto::read(*store, from); SubstituteFlag substitute = NoSubstitute; if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { @@ -294,7 +300,7 @@ } auto res = store->queryValidPaths(paths, substitute); logger->stopWork(); - worker_proto::write(*store, to, res); + workerProtoWrite(*store, to, res); break; } @@ -310,11 +316,11 @@ } case wopQuerySubstitutablePaths: { - auto paths = worker_proto::read(*store, from, Phantom {}); + auto paths = WorkerProto::read(*store, from); logger->startWork(); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - worker_proto::write(*store, to, res); + workerProtoWrite(*store, to, res); break; } @@ -343,7 +349,7 @@ paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - worker_proto::write(*store, to, paths); + workerProtoWrite(*store, to, paths); break; } @@ -361,7 +367,7 @@ logger->startWork(); auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); - worker_proto::write(*store, to, outputs); + workerProtoWrite(*store, to, outputs); break; } @@ -387,7 +393,7 @@ if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { auto name = readString(from); auto camStr = readString(from); - auto refs = worker_proto::read(*store, from, Phantom {}); + auto refs = WorkerProto::read(*store, from); bool repairBool; from >> repairBool; auto repair = RepairFlag{repairBool}; @@ -395,21 +401,25 @@ logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); + auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr); + auto hashType = hashType_; // work around clang bug FramedSource source(from); // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { - [&](TextHashMethod &) { + [&](const TextIngestionMethod &) { + if (hashType != htSHA256) + throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", + name, printHashType(hashType)); // We could stream this by changing Store std::string contents = source.drain(); auto path = store->addTextToStore(name, contents, refs, repair); return store->queryPathInfo(path); }, - [&](FixedOutputHashMethod & fohm) { - auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); + [&](const FileIngestionMethod & fim) { + auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs); return store->queryPathInfo(path); }, - }, contentAddressMethod); + }, contentAddressMethod.raw); }(); logger->stopWork(); @@ -485,7 +495,7 @@ case wopAddTextToStore: { std::string suffix = readString(from); std::string s = readString(from); - auto refs = worker_proto::read(*store, from, Phantom {}); + auto refs = WorkerProto::read(*store, from); logger->startWork(); auto path = store->addTextToStore(suffix, s, refs, NoRepair); logger->stopWork(); @@ -523,7 +533,14 @@ mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation in this message: the daemon code (i.e. + this file) knows whether a client/connection is trusted, but it + does not how how the client was authenticated. The mechanism + need not be getting the UID of the other end of a Unix Domain + Socket. + */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); } @@ -540,7 +557,9 @@ mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation; see above. */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); @@ -548,7 +567,7 @@ auto results = store->buildPathsWithResults(drvs, mode); logger->stopWork(); - worker_proto::write(*store, to, results); + workerProtoWrite(*store, to, results); break; } @@ -622,7 +641,10 @@ to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; } if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { - worker_proto::write(*store, to, res.builtOutputs); + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + workerProtoWrite(*store, to, builtOutputs); } break; } @@ -687,7 +709,7 @@ case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = worker_proto::read(*store, from, Phantom {}); + options.pathsToDelete = WorkerProto::read(*store, from); from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); @@ -757,7 +779,7 @@ else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.references); + workerProtoWrite(*store, to, i->second.references); to << i->second.downloadSize << i->second.narSize; } @@ -768,11 +790,11 @@ SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = worker_proto::read(*store, from, Phantom {}); + auto paths = WorkerProto::read(*store, from); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); } else - pathsMap = worker_proto::read(*store, from, Phantom {}); + pathsMap = WorkerProto::read(*store, from); logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); @@ -780,7 +802,7 @@ for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.references); + workerProtoWrite(*store, to, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; @@ -790,7 +812,7 @@ logger->startWork(); auto paths = store->queryAllValidPaths(); logger->stopWork(); - worker_proto::write(*store, to, paths); + workerProtoWrite(*store, to, paths); break; } @@ -862,10 +884,10 @@ ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, from, Phantom {}); + info.references = WorkerProto::read(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = parseContentAddressOpt(readString(from)); + info.ca = ContentAddress::parseOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; @@ -913,9 +935,9 @@ uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - worker_proto::write(*store, to, willBuild); - worker_proto::write(*store, to, willSubstitute); - worker_proto::write(*store, to, unknown); + workerProtoWrite(*store, to, willBuild); + workerProtoWrite(*store, to, willSubstitute); + workerProtoWrite(*store, to, unknown); to << downloadSize << narSize; break; } @@ -928,7 +950,7 @@ store->registerDrvOutput(Realisation{ .id = outputId, .outPath = outputPath}); } else { - auto realisation = worker_proto::read(*store, from, Phantom()); + auto realisation = WorkerProto::read(*store, from); store->registerDrvOutput(realisation); } logger->stopWork(); @@ -943,11 +965,11 @@ if (GET_PROTOCOL_MINOR(clientVersion) < 31) { std::set outPaths; if (info) outPaths.insert(info->outPath); - worker_proto::write(*store, to, outPaths); + workerProtoWrite(*store, to, outPaths); } else { std::set realisations; if (info) realisations.insert(*info); - worker_proto::write(*store, to, realisations); + workerProtoWrite(*store, to, realisations); } break; } @@ -979,8 +1001,7 @@ FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - std::function authHook) + RecursiveFlag recursive) { auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; @@ -1018,15 +1039,20 @@ if (GET_PROTOCOL_MINOR(clientVersion) >= 33) to << nixVersion; + if (GET_PROTOCOL_MINOR(clientVersion) >= 35) { + // We and the underlying store both need to trust the client for + // it to be trusted. + auto temp = trusted + ? store->isTrustedClient() + : std::optional { NotTrusted }; + workerProtoWrite(*store, to, temp); + } + /* Send startup error messages to the client. */ tunnelLogger->startWork(); try { - /* If we can't accept clientVersion, then throw an error - *here* (not above). */ - authHook(*store); - tunnelLogger->stopWork(); to.flush(); @@ -1045,6 +1071,8 @@ opCount++; + debug("performing daemon worker op: %d", op); + try { performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op); } catch (Error & e) { diff -Nru nix-2.8.0/src/libstore/daemon.hh nix-2.16.1+dfsg/src/libstore/daemon.hh --- nix-2.8.0/src/libstore/daemon.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/daemon.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,11 +1,11 @@ #pragma once +///@file #include "serialise.hh" #include "store-api.hh" namespace nix::daemon { -enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; void processConnection( @@ -13,11 +13,6 @@ FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - /* Arbitrary hook to check authorization / initialize user data / whatever - after the protocol has been negotiated. The idea is that this function - and everything it calls doesn't know about this stuff, and the - `nix-daemon` handles that instead. */ - std::function authHook); + RecursiveFlag recursive); } diff -Nru nix-2.8.0/src/libstore/derivations.cc nix-2.16.1+dfsg/src/libstore/derivations.cc --- nix-2.8.0/src/libstore/derivations.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/derivations.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,10 +1,13 @@ #include "derivations.hh" +#include "downstream-placeholder.hh" #include "store-api.hh" #include "globals.hh" #include "util.hh" +#include "split.hh" #include "worker-protocol.hh" #include "fs-accessor.hh" #include +#include namespace nix { @@ -34,9 +37,9 @@ StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { - return store.makeFixedOutputPath( - hash.method, hash.hash, - outputPathName(drvName, outputName)); + return store.makeFixedOutputPathFromCA( + outputPathName(drvName, outputName), + ContentAddressWithReferences::withoutRefs(ca)); } @@ -210,32 +213,30 @@ static DerivationOutput parseDerivationOutput(const Store & store, - std::string_view pathS, std::string_view hashAlgo, std::string_view hash) + std::string_view pathS, std::string_view hashAlgo, std::string_view hashS) { if (hashAlgo != "") { - auto method = FileIngestionMethod::Flat; - if (hashAlgo.substr(0, 2) == "r:") { - method = FileIngestionMethod::Recursive; - hashAlgo = hashAlgo.substr(2); - } + ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo); + if (method == TextIngestionMethod {}) + experimentalFeatureSettings.require(Xp::DynamicDerivations); const auto hashType = parseHashType(hashAlgo); - if (hash == "impure") { - settings.requireExperimentalFeature(Xp::ImpureDerivations); + if (hashS == "impure") { + experimentalFeatureSettings.require(Xp::ImpureDerivations); assert(pathS == ""); return DerivationOutput::Impure { .method = std::move(method), .hashType = std::move(hashType), }; - } else if (hash != "") { + } else if (hashS != "") { validatePath(pathS); + auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType); return DerivationOutput::CAFixed { - .hash = FixedOutputHash { - .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed(hash, hashType), - }, + .ca = ContentAddress::fromParts( + std::move(method), + std::move(hash)), }; } else { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); assert(pathS == ""); return DerivationOutput::CAFloating { .method = std::move(method), @@ -312,6 +313,15 @@ } +/** + * Print a derivation string literal to an `std::string`. + * + * This syntax does not generalize to the expression language, which needs to + * escape `$`. + * + * @param res Where to print to + * @param s Which logical string to print + */ static void printString(std::string & res, std::string_view s) { boost::container::small_vector buffer; @@ -383,12 +393,12 @@ }, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); - s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); + s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); + s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType)); s += ','; printUnquotedString(s, ""); }, [&](const DerivationOutput::Deferred &) { @@ -399,7 +409,7 @@ [&](const DerivationOutputImpure & doi) { // FIXME s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); + s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType)); s += ','; printUnquotedString(s, "impure"); } }, i.second.raw()); @@ -448,7 +458,7 @@ // FIXME: remove -bool isDerivation(const std::string & fileName) +bool isDerivation(std::string_view fileName) { return hasSuffix(fileName, drvExtension); } @@ -616,8 +626,8 @@ for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw()); auto hash = hashString(htSHA256, "fixed:out:" - + dof.hash.printMethodAlgo() + ":" - + dof.hash.hash.to_string(Base16, false) + ":" + + dof.ca.printMethodAlgo() + ":" + + dof.ca.getHash().to_string(Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -661,8 +671,10 @@ if (res.kind == DrvHash::Kind::Deferred) kind = DrvHash::Kind::Deferred; for (auto & outputName : inputOutputs) { - const auto h = res.hashes.at(outputName); - inputs2[h.to_string(Base16, false)].insert(outputName); + const auto h = get(res.hashes, outputName); + if (!h) + throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); + inputs2[h->to_string(Base16, false)].insert(outputName); } } @@ -686,12 +698,6 @@ } -bool wantOutput(const std::string & output, const std::set & wanted) -{ - return wanted.empty() || wanted.find(output) != wanted.end(); -} - - static DerivationOutput readDerivationOutput(Source & in, const Store & store) { const auto pathS = readString(in); @@ -743,7 +749,7 @@ drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = worker_proto::read(store, in, Phantom {}); + drv.inputSrcs = WorkerProto::read(store, in); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -771,12 +777,12 @@ }, [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) - << dof.hash.printMethodAlgo() - << dof.hash.hash.to_string(Base16, false); + << dof.ca.printMethodAlgo() + << dof.ca.getHash().to_string(Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" - << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) + << (dof.method.renderPrefix() + printHashType(dof.hashType)) << ""; }, [&](const DerivationOutput::Deferred &) { @@ -786,12 +792,12 @@ }, [&](const DerivationOutput::Impure & doi) { out << "" - << (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)) + << (doi.method.renderPrefix() + printHashType(doi.hashType)) << "impure"; }, }, i.second.raw()); } - worker_proto::write(store, out, drv.inputSrcs); + workerProtoWrite(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) @@ -805,13 +811,7 @@ return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); } -std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName) -{ - auto drvNameWithExtension = drvPath.name(); - auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); - auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); - return "/" + hashString(htSHA256, clearText).to_string(Base32, false); -} + static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) @@ -836,8 +836,11 @@ auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { if (std::holds_alternative(output.raw())) { - auto & h = hashModulo.hashes.at(outputName); - auto outPath = store.makeOutputPath(outputName, h, drv.name); + auto h = get(hashModulo.hashes, outputName); + if (!h) + throw Error("derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", + drv.name, outputName); + auto outPath = store.makeOutputPath(outputName, *h, drv.name); drv.env[outputName] = store.printStorePath(outPath); output = DerivationOutput::InputAddressed { .path = std::move(outPath), @@ -872,7 +875,7 @@ for (auto & outputName : inputOutputs) { if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { inputRewrites.emplace( - downstreamPlaceholder(store, inputDrv, outputName), + DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), store.printStorePath(*actualPath)); resolved.inputSrcs.insert(*actualPath); } else { @@ -889,6 +892,237 @@ return resolved; } + +void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const +{ + assert(drvPath.isDerivation()); + std::string drvName(drvPath.name()); + drvName = drvName.substr(0, drvName.size() - drvExtension.size()); + + if (drvName != name) { + throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name); + } + + auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) + { + auto j = env.find(varName); + if (j == env.end() || store.parseStorePath(j->second) != actual) + throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", + store.printStorePath(drvPath), varName, store.printStorePath(actual)); + }; + + + // Don't need the answer, but do this anyways to assert is proper + // combination. The code below is more general and naturally allows + // combinations that are currently prohibited. + type(); + + std::optional hashesModulo; + for (auto & i : outputs) { + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doia) { + if (!hashesModulo) { + // somewhat expensive so we do lazily + hashesModulo = hashDerivationModulo(store, *this, true); + } + auto currentOutputHash = get(hashesModulo->hashes, i.first); + if (!currentOutputHash) + throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), i.first); + StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName); + if (doia.path != recomputed) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed)); + envHasRightPath(doia.path, i.first); + }, + [&](const DerivationOutput::CAFixed & dof) { + auto path = dof.path(store, drvName, i.first); + envHasRightPath(path, i.first); + }, + [&](const DerivationOutput::CAFloating &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Deferred &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Impure &) { + /* Nothing to check */ + }, + }, i.second.raw()); + } +} + + const Hash impureOutputHash = hashString(htSHA256, "impure"); +nlohmann::json DerivationOutput::toJSON( + const Store & store, std::string_view drvName, std::string_view outputName) const +{ + nlohmann::json res = nlohmann::json::object(); + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doi) { + res["path"] = store.printStorePath(doi.path); + }, + [&](const DerivationOutput::CAFixed & dof) { + res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); + res["hashAlgo"] = dof.ca.printMethodAlgo(); + res["hash"] = dof.ca.getHash().to_string(Base16, false); + // FIXME print refs? + }, + [&](const DerivationOutput::CAFloating & dof) { + res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType); + }, + [&](const DerivationOutput::Deferred &) {}, + [&](const DerivationOutput::Impure & doi) { + res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType); + res["impure"] = true; + }, + }, raw()); + return res; +} + + +DerivationOutput DerivationOutput::fromJSON( + const Store & store, std::string_view drvName, std::string_view outputName, + const nlohmann::json & _json, + const ExperimentalFeatureSettings & xpSettings) +{ + std::set keys; + auto json = (std::map) _json; + + for (const auto & [key, _] : json) + keys.insert(key); + + auto methodAlgo = [&]() -> std::pair { + std::string hashAlgo = json["hashAlgo"]; + // remaining to parse, will be mutated by parsers + std::string_view s = hashAlgo; + ContentAddressMethod method = ContentAddressMethod::parsePrefix(s); + if (method == TextIngestionMethod {}) + xpSettings.require(Xp::DynamicDerivations); + auto hashType = parseHashType(s); + return { std::move(method), std::move(hashType) }; + }; + + if (keys == (std::set { "path" })) { + return DerivationOutput::InputAddressed { + .path = store.parseStorePath((std::string) json["path"]), + }; + } + + else if (keys == (std::set { "path", "hashAlgo", "hash" })) { + auto [method, hashType] = methodAlgo(); + auto dof = DerivationOutput::CAFixed { + .ca = ContentAddress::fromParts( + std::move(method), + Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)), + }; + if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) + throw Error("Path doesn't match derivation output"); + return dof; + } + + else if (keys == (std::set { "hashAlgo" })) { + xpSettings.require(Xp::CaDerivations); + auto [method, hashType] = methodAlgo(); + return DerivationOutput::CAFloating { + .method = std::move(method), + .hashType = std::move(hashType), + }; + } + + else if (keys == (std::set { })) { + return DerivationOutput::Deferred {}; + } + + else if (keys == (std::set { "hashAlgo", "impure" })) { + xpSettings.require(Xp::ImpureDerivations); + auto [method, hashType] = methodAlgo(); + return DerivationOutput::Impure { + .method = std::move(method), + .hashType = hashType, + }; + } + + else { + throw Error("invalid JSON for derivation output"); + } +} + + +nlohmann::json Derivation::toJSON(const Store & store) const +{ + nlohmann::json res = nlohmann::json::object(); + + res["name"] = name; + + { + nlohmann::json & outputsObj = res["outputs"]; + outputsObj = nlohmann::json::object(); + for (auto & [outputName, output] : outputs) { + outputsObj[outputName] = output.toJSON(store, name, outputName); + } + } + + { + auto& inputsList = res["inputSrcs"]; + inputsList = nlohmann::json ::array(); + for (auto & input : inputSrcs) + inputsList.emplace_back(store.printStorePath(input)); + } + + { + auto& inputDrvsObj = res["inputDrvs"]; + inputDrvsObj = nlohmann::json ::object(); + for (auto & input : inputDrvs) + inputDrvsObj[store.printStorePath(input.first)] = input.second; + } + + res["system"] = platform; + res["builder"] = builder; + res["args"] = args; + res["env"] = env; + + return res; +} + + +Derivation Derivation::fromJSON( + const Store & store, + const nlohmann::json & json) +{ + Derivation res; + + res.name = json["name"]; + + { + auto & outputsObj = json["outputs"]; + for (auto & [outputName, output] : outputsObj.items()) { + res.outputs.insert_or_assign( + outputName, + DerivationOutput::fromJSON(store, res.name, outputName, output)); + } + } + + { + auto & inputsList = json["inputSrcs"]; + for (auto & input : inputsList) + res.inputSrcs.insert(store.parseStorePath(static_cast(input))); + } + + { + auto & inputDrvsObj = json["inputDrvs"]; + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + res.inputDrvs[store.parseStorePath(inputDrvPath)] = + static_cast(inputOutputs); + } + + res.platform = json["system"]; + res.builder = json["builder"]; + res.args = json["args"]; + res.env = json["env"]; + + return res; +} + } diff -Nru nix-2.8.0/src/libstore/derivations.hh nix-2.16.1+dfsg/src/libstore/derivations.hh --- nix-2.8.0/src/libstore/derivations.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/derivations.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,11 +1,14 @@ #pragma once +///@file #include "path.hh" #include "types.hh" #include "hash.hh" #include "content-address.hh" #include "repair-flag.hh" +#include "derived-path.hh" #include "sync.hh" +#include "comparator.hh" #include #include @@ -13,46 +16,89 @@ namespace nix { +class Store; /* Abstract syntax of derivations. */ -/* The traditional non-fixed-output derivation type. */ +/** + * The traditional non-fixed-output derivation type. + */ struct DerivationOutputInputAddressed { StorePath path; + + GENERATE_CMP(DerivationOutputInputAddressed, me->path); }; -/* Fixed-output derivations, whose output paths are content addressed - according to that fixed output. */ +/** + * Fixed-output derivations, whose output paths are content + * addressed according to that fixed output. + */ struct DerivationOutputCAFixed { - FixedOutputHash hash; /* hash used for expected hash computation */ + /** + * Method and hash used for expected hash computation. + * + * References are not allowed by fiat. + */ + ContentAddress ca; + + /** + * Return the \ref StorePath "store path" corresponding to this output + * + * @param drvName The name of the derivation this is an output of, without the `.drv`. + * @param outputName The name of this output. + */ StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; + + GENERATE_CMP(DerivationOutputCAFixed, me->ca); }; -/* Floating-output derivations, whose output paths are content addressed, but - not fixed, and so are dynamically calculated from whatever the output ends - up being. */ +/** + * Floating-output derivations, whose output paths are content + * addressed, but not fixed, and so are dynamically calculated from + * whatever the output ends up being. + * */ struct DerivationOutputCAFloating { - /* information used for expected hash computation */ - FileIngestionMethod method; + /** + * How the file system objects will be serialized for hashing + */ + ContentAddressMethod method; + + /** + * How the serialization will be hashed + */ HashType hashType; + + GENERATE_CMP(DerivationOutputCAFloating, me->method, me->hashType); }; -/* Input-addressed output which depends on a (CA) derivation whose hash isn't - * known yet. +/** + * Input-addressed output which depends on a (CA) derivation whose hash + * isn't known yet. */ -struct DerivationOutputDeferred {}; +struct DerivationOutputDeferred { + GENERATE_CMP(DerivationOutputDeferred); +}; -/* Impure output which is moved to a content-addressed location (like - CAFloating) but isn't registered as a realization. +/** + * Impure output which is moved to a content-addressed location (like + * CAFloating) but isn't registered as a realization. */ struct DerivationOutputImpure { - /* information used for expected hash computation */ - FileIngestionMethod method; + /** + * How the file system objects will be serialized for hashing + */ + ContentAddressMethod method; + + /** + * How the serialization will be hashed + */ HashType hashType; + + GENERATE_CMP(DerivationOutputImpure, me->method, me->hashType); }; typedef std::variant< @@ -63,6 +109,9 @@ DerivationOutputImpure > _DerivationOutputRaw; +/** + * A single output of a BasicDerivation (and Derivation). + */ struct DerivationOutput : _DerivationOutputRaw { using Raw = _DerivationOutputRaw; @@ -74,38 +123,90 @@ using Deferred = DerivationOutputDeferred; using Impure = DerivationOutputImpure; - /* Note, when you use this function you should make sure that you're passing - the right derivation name. When in doubt, you should use the safer - interface provided by BasicDerivation::outputsAndOptPaths */ + /** + * \note when you use this function you should make sure that you're + * passing the right derivation name. When in doubt, you should use + * the safer interface provided by + * BasicDerivation::outputsAndOptPaths + */ std::optional path(const Store & store, std::string_view drvName, std::string_view outputName) const; inline const Raw & raw() const { return static_cast(*this); } + + nlohmann::json toJSON( + const Store & store, + std::string_view drvName, + std::string_view outputName) const; + /** + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DerivationOutput fromJSON( + const Store & store, + std::string_view drvName, + std::string_view outputName, + const nlohmann::json & json, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); }; typedef std::map DerivationOutputs; -/* These are analogues to the previous DerivationOutputs data type, but they - also contains, for each output, the (optional) store path in which it would - be written. To calculate values of these types, see the corresponding - functions in BasicDerivation */ +/** + * These are analogues to the previous DerivationOutputs data type, + * but they also contains, for each output, the (optional) store + * path in which it would be written. To calculate values of these + * types, see the corresponding functions in BasicDerivation. + */ typedef std::map>> DerivationOutputsAndOptPaths; -/* For inputs that are sub-derivations, we specify exactly which - output IDs we are interested in. */ +/** + * For inputs that are sub-derivations, we specify exactly which + * output IDs we are interested in. + */ typedef std::map DerivationInputs; +/** + * Input-addressed derivation types + */ struct DerivationType_InputAddressed { + /** + * True iff the derivation type can't be determined statically, + * for instance because it (transitively) depends on a content-addressed + * derivation. + */ bool deferred; }; +/** + * Content-addressed derivation types + */ struct DerivationType_ContentAddressed { + /** + * Whether the derivation should be built safely inside a sandbox. + */ bool sandboxed; + /** + * Whether the derivation's outputs' content-addresses are "fixed" + * or "floating. + * + * - Fixed: content-addresses are written down as part of the + * derivation itself. If the outputs don't end up matching the + * build fails. + * + * - Floating: content-addresses are not written down, we do not + * know them until we perform the build. + */ bool fixed; }; +/** + * Impure derivation type + * + * This is similar at buil-time to the content addressed, not standboxed, not fixed + * type, but has some restrictions on its usage. + */ struct DerivationType_Impure { }; @@ -122,30 +223,38 @@ using ContentAddressed = DerivationType_ContentAddressed; using Impure = DerivationType_Impure; - /* Do the outputs of the derivation have paths calculated from their content, - or from the derivation itself? */ + /** + * Do the outputs of the derivation have paths calculated from their + * content, or from the derivation itself? + */ bool isCA() const; - /* Is the content of the outputs fixed a-priori via a hash? Never true for - non-CA derivations. */ + /** + * Is the content of the outputs fixed a priori via a hash? + * Never true for non-CA derivations. + */ bool isFixed() const; - /* Whether the derivation is fully sandboxed. If false, the - sandbox is opened up, e.g. the derivation has access to the - network. Note that whether or not we actually sandbox the - derivation is controlled separately. Always true for non-CA - derivations. */ + /** + * Whether the derivation is fully sandboxed. If false, the sandbox + * is opened up, e.g. the derivation has access to the network. Note + * that whether or not we actually sandbox the derivation is + * controlled separately. Always true for non-CA derivations. + */ bool isSandboxed() const; - /* Whether the derivation is expected to produce the same result - every time, and therefore it only needs to be built once. This - is only false for derivations that have the attribute '__impure - = true'. */ + /** + * Whether the derivation is expected to produce the same result + * every time, and therefore it only needs to be built once. This is + * only false for derivations that have the attribute '__impure = + * true'. + */ bool isPure() const; - /* Does the derivation knows its own output paths? - Only true when there's no floating-ca derivation involved in the - closure, or if fixed output. + /** + * Does the derivation knows its own output paths? + * Only true when there's no floating-ca derivation involved in the + * closure, or if fixed output. */ bool hasKnownOutputPaths() const; @@ -156,8 +265,14 @@ struct BasicDerivation { - DerivationOutputs outputs; /* keyed on symbolic IDs */ - StorePathSet inputSrcs; /* inputs that are sources */ + /** + * keyed on symbolic IDs + */ + DerivationOutputs outputs; + /** + * inputs that are sources + */ + StorePathSet inputSrcs; std::string platform; Path builder; Strings args; @@ -169,156 +284,218 @@ bool isBuiltin() const; - /* Return true iff this is a fixed-output derivation. */ + /** + * Return true iff this is a fixed-output derivation. + */ DerivationType type() const; - /* Return the output names of a derivation. */ + /** + * Return the output names of a derivation. + */ StringSet outputNames() const; - /* Calculates the maps that contains all the DerivationOutputs, but - augmented with knowledge of the Store paths they would be written - into. */ + /** + * Calculates the maps that contains all the DerivationOutputs, but + * augmented with knowledge of the Store paths they would be written + * into. + */ DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; static std::string_view nameFromPath(const StorePath & storePath); + + GENERATE_CMP(BasicDerivation, + me->outputs, + me->inputSrcs, + me->platform, + me->builder, + me->args, + me->env, + me->name); }; struct Derivation : BasicDerivation { - DerivationInputs inputDrvs; /* inputs that are sub-derivations */ + /** + * inputs that are sub-derivations + */ + DerivationInputs inputDrvs; - /* Print a derivation. */ + /** + * Print a derivation. + */ std::string unparse(const Store & store, bool maskOutputs, std::map * actualInputs = nullptr) const; - /* Return the underlying basic derivation but with these changes: - - 1. Input drvs are emptied, but the outputs of them that were used are - added directly to input sources. - - 2. Input placeholders are replaced with realized input store paths. */ + /** + * Return the underlying basic derivation but with these changes: + * + * 1. Input drvs are emptied, but the outputs of them that were used + * are added directly to input sources. + * + * 2. Input placeholders are replaced with realized input store + * paths. + */ std::optional tryResolve(Store & store) const; - /* Like the above, but instead of querying the Nix database for - realisations, uses a given mapping from input derivation paths - + output names to actual output store paths. */ + /** + * Like the above, but instead of querying the Nix database for + * realisations, uses a given mapping from input derivation paths + + * output names to actual output store paths. + */ std::optional tryResolve( Store & store, const std::map, StorePath> & inputDrvOutputs) const; + /** + * Check that the derivation is valid and does not present any + * illegal states. + * + * This is mainly a matter of checking the outputs, where our C++ + * representation supports all sorts of combinations we do not yet + * allow. + */ + void checkInvariants(Store & store, const StorePath & drvPath) const; + Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } + + nlohmann::json toJSON(const Store & store) const; + static Derivation fromJSON( + const Store & store, + const nlohmann::json & json); + + GENERATE_CMP(Derivation, + static_cast(*me), + me->inputDrvs); }; class Store; -/* Write a derivation to the Nix store, and return its path. */ +/** + * Write a derivation to the Nix store, and return its path. + */ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair = NoRepair, bool readOnly = false); -/* Read a derivation from a file. */ +/** + * Read a derivation from a file. + */ Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); -// FIXME: remove -bool isDerivation(const std::string & fileName); - -/* Calculate the name that will be used for the store path for this - output. +/** + * \todo Remove. + * + * Use Path::isDerivation instead. + */ +bool isDerivation(std::string_view fileName); - This is usually -, but is just when - the output name is "out". */ +/** + * Calculate the name that will be used for the store path for this + * output. + * + * This is usually -, but is just when + * the output name is "out". + */ std::string outputPathName(std::string_view drvName, std::string_view outputName); -// The hashes modulo of a derivation. -// -// Each output is given a hash, although in practice only the content-addressed -// derivations (fixed-output or not) will have a different hash for each -// output. +/** + * The hashes modulo of a derivation. + * + * Each output is given a hash, although in practice only the content-addressed + * derivations (fixed-output or not) will have a different hash for each + * output. + */ struct DrvHash { + /** + * Map from output names to hashes + */ std::map hashes; enum struct Kind : bool { - // Statically determined derivations. - // This hash will be directly used to compute the output paths + /** + * Statically determined derivations. + * This hash will be directly used to compute the output paths + */ Regular, - // Floating-output derivations (and their reverse dependencies). + + /** + * Floating-output derivations (and their reverse dependencies). + */ Deferred, }; + /** + * The kind of derivation this is, simplified for just "derivation hash + * modulo" purposes. + */ Kind kind; }; void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; -/* Returns hashes with the details of fixed-output subderivations - expunged. - - A fixed-output derivation is a derivation whose outputs have a - specified content hash and hash algorithm. (Currently they must have - exactly one output (`out'), which is specified using the `outputHash' - and `outputHashAlgo' attributes, but the algorithm doesn't assume - this.) We don't want changes to such derivations to propagate upwards - through the dependency graph, changing output paths everywhere. - - For instance, if we change the url in a call to the `fetchurl' - function, we do not want to rebuild everything depending on it---after - all, (the hash of) the file being downloaded is unchanged. So the - *output paths* should not change. On the other hand, the *derivation - paths* should change to reflect the new dependency graph. - - For fixed-output derivations, this returns a map from the name of - each output to its hash, unique up to the output's contents. - - For regular derivations, it returns a single hash of the derivation - ATerm, after subderivations have been likewise expunged from that - derivation. +/** + * Returns hashes with the details of fixed-output subderivations + * expunged. + * + * A fixed-output derivation is a derivation whose outputs have a + * specified content hash and hash algorithm. (Currently they must have + * exactly one output (`out`), which is specified using the `outputHash` + * and `outputHashAlgo` attributes, but the algorithm doesn't assume + * this.) We don't want changes to such derivations to propagate upwards + * through the dependency graph, changing output paths everywhere. + * + * For instance, if we change the url in a call to the `fetchurl` + * function, we do not want to rebuild everything depending on it---after + * all, (the hash of) the file being downloaded is unchanged. So the + * *output paths* should not change. On the other hand, the *derivation + * paths* should change to reflect the new dependency graph. + * + * For fixed-output derivations, this returns a map from the name of + * each output to its hash, unique up to the output's contents. + * + * For regular derivations, it returns a single hash of the derivation + * ATerm, after subderivations have been likewise expunged from that + * derivation. */ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); -/* - Return a map associating each output to a hash that uniquely identifies its - derivation (modulo the self-references). - - FIXME: what is the Hash in this map? +/** + * Return a map associating each output to a hash that uniquely identifies its + * derivation (modulo the self-references). + * + * \todo What is the Hash in this map? */ std::map staticOutputHashes(Store & store, const Derivation & drv); -/* Memoisation of hashDerivationModulo(). */ +/** + * Memoisation of hashDerivationModulo(). + */ typedef std::map DrvHashes; // FIXME: global, though at least thread-safe. extern Sync drvHashes; -bool wantOutput(const std::string & output, const std::set & wanted); - struct Source; struct Sink; Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); -/* This creates an opaque and almost certainly unique string - deterministically from the output name. - - It is used as a placeholder to allow derivations to refer to their - own outputs without needing to use the hash of a derivation in - itself, making the hash near-impossible to calculate. */ +/** + * This creates an opaque and almost certainly unique string + * deterministically from the output name. + * + * It is used as a placeholder to allow derivations to refer to their + * own outputs without needing to use the hash of a derivation in + * itself, making the hash near-impossible to calculate. + */ std::string hashPlaceholder(const std::string_view outputName); -/* This creates an opaque and almost certainly unique string - deterministically from a derivation path and output name. - - It is used as a placeholder to allow derivations to refer to - content-addressed paths whose content --- and thus the path - themselves --- isn't yet known. This occurs when a derivation has a - dependency which is a CA derivation. */ -std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName); - extern const Hash impureOutputHash; } diff -Nru nix-2.8.0/src/libstore/derived-path.cc nix-2.16.1+dfsg/src/libstore/derived-path.cc --- nix-2.8.0/src/libstore/derived-path.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/derived-path.cc 2023-06-06 10:55:23.000000000 +0000 @@ -4,6 +4,8 @@ #include +#include + namespace nix { nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { @@ -17,10 +19,11 @@ res["drvPath"] = store->printStorePath(drvPath); // Fallback for the input-addressed derivation case: We expect to always be // able to print the output paths, so let’s do it - auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath); - for (const auto& output : outputs) { - if (knownOutputs.at(output)) - res["outputs"][output] = store->printStorePath(knownOutputs.at(output).value()); + const auto outputMap = store->queryPartialDerivationOutputMap(drvPath); + for (const auto & [output, outputPathOpt] : outputMap) { + if (!outputs.contains(output)) continue; + if (outputPathOpt) + res["outputs"][output] = store->printStorePath(*outputPathOpt); else res["outputs"][output] = nullptr; } @@ -51,38 +54,39 @@ ); } -template -nlohmann::json stuffToJSON(const std::vector & ts, ref store) { - auto res = nlohmann::json::array(); - for (const T & t : ts) { - std::visit([&res, store](const auto & t) { - res.push_back(t.toJSON(store)); - }, t.raw()); - } - return res; +std::string DerivedPath::Opaque::to_string(const Store & store) const +{ + return store.printStorePath(path); } -nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store) -{ return stuffToJSON(buildables, store); } -nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) -{ return stuffToJSON(paths, store); } - - -std::string DerivedPath::Opaque::to_string(const Store & store) const { - return store.printStorePath(path); +std::string DerivedPath::Built::to_string(const Store & store) const +{ + return store.printStorePath(drvPath) + + '^' + + outputs.to_string(); } -std::string DerivedPath::Built::to_string(const Store & store) const { +std::string DerivedPath::Built::to_string_legacy(const Store & store) const +{ return store.printStorePath(drvPath) - + "!" - + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); + + '!' + + outputs.to_string(); } std::string DerivedPath::to_string(const Store & store) const { - return std::visit( - [&](const auto & req) { return req.to_string(store); }, - this->raw()); + return std::visit(overloaded { + [&](const DerivedPath::Built & req) { return req.to_string(store); }, + [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); +} + +std::string DerivedPath::to_string_legacy(const Store & store) const +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); }, + [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); } @@ -91,24 +95,30 @@ return {store.parseStorePath(s)}; } -DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view s) +DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS) { - size_t n = s.find("!"); - assert(n != s.npos); - auto drvPath = store.parseStorePath(s.substr(0, n)); - auto outputsS = s.substr(n + 1); - std::set outputs; - if (outputsS != "*") - outputs = tokenizeString>(outputsS, ","); - return {drvPath, outputs}; + return { + .drvPath = store.parseStorePath(drvS), + .outputs = OutputsSpec::parse(outputsS), + }; } -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator) { - size_t n = s.find("!"); + size_t n = s.find(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) - : (DerivedPath) DerivedPath::Built::parse(store, s); + : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); +} + +DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +{ + return parseWith(store, s, "^"); +} + +DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) +{ + return parseWith(store, s, "!"); } RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const @@ -121,12 +131,17 @@ auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath)); for (auto& [outputName, outputPath] : p.outputs) { - if (settings.isExperimentalFeatureEnabled( + if (experimentalFeatureSettings.isEnabled( Xp::CaDerivations)) { + auto drvOutput = get(drvHashes, outputName); + if (!drvOutput) + throw Error( + "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", + store.printStorePath(p.drvPath), outputName); auto thisRealisation = store.queryRealisation( - DrvOutput{drvHashes.at(outputName), outputName}); - assert(thisRealisation); // We’ve built it, so we must h - // ve the realisation + DrvOutput{*drvOutput, outputName}); + assert(thisRealisation); // We’ve built it, so we must + // have the realisation res.insert(*thisRealisation); } else { res.insert(outputPath); diff -Nru nix-2.8.0/src/libstore/derived-path.hh nix-2.16.1+dfsg/src/libstore/derived-path.hh --- nix-2.8.0/src/libstore/derived-path.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/derived-path.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,10 +1,13 @@ #pragma once +///@file #include "util.hh" #include "path.hh" #include "realisation.hh" +#include "outputs-spec.hh" +#include "comparator.hh" -#include +#include #include @@ -26,8 +29,7 @@ std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); - bool operator < (const DerivedPathOpaque & b) const - { return path < b.path; } + GENERATE_CMP(DerivedPathOpaque, me->path); }; /** @@ -44,14 +46,23 @@ */ struct DerivedPathBuilt { StorePath drvPath; - std::set outputs; + OutputsSpec outputs; + /** + * Uses `^` as the separator + */ std::string to_string(const Store & store) const; - static DerivedPathBuilt parse(const Store & store, std::string_view); + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * The caller splits on the separator, so it works for both variants. + */ + static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs); nlohmann::json toJSON(ref store) const; - bool operator < (const DerivedPathBuilt & b) const - { return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); } + GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); }; using _DerivedPathRaw = std::variant< @@ -80,8 +91,22 @@ return static_cast(*this); } + /** + * Uses `^` as the separator + */ std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * Uses `^` as the separator + */ static DerivedPath parse(const Store & store, std::string_view); + /** + * Uses `!` as the separator + */ + static DerivedPath parseLegacy(const Store & store, std::string_view); }; /** @@ -95,6 +120,8 @@ nlohmann::json toJSON(ref store) const; static BuiltPathBuilt parse(const Store & store, std::string_view); + + GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); }; using _BuiltPathRaw = std::variant< @@ -103,7 +130,7 @@ >; /** - * A built path. Similar to a `DerivedPath`, but enriched with the corresponding + * A built path. Similar to a DerivedPath, but enriched with the corresponding * output path(s). */ struct BuiltPath : _BuiltPathRaw { @@ -125,7 +152,4 @@ typedef std::vector DerivedPaths; typedef std::vector BuiltPaths; -nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store); -nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref store); - } diff -Nru nix-2.8.0/src/libstore/downstream-placeholder.cc nix-2.16.1+dfsg/src/libstore/downstream-placeholder.cc --- nix-2.8.0/src/libstore/downstream-placeholder.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/downstream-placeholder.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,39 @@ +#include "downstream-placeholder.hh" +#include "derivations.hh" + +namespace nix { + +std::string DownstreamPlaceholder::render() const +{ + return "/" + hash.to_string(Base32, false); +} + + +DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( + const StorePath & drvPath, + std::string_view outputName) +{ + auto drvNameWithExtension = drvPath.name(); + auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); + auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); + return DownstreamPlaceholder { + hashString(htSHA256, clearText) + }; +} + +DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( + const DownstreamPlaceholder & placeholder, + std::string_view outputName, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::DynamicDerivations); + auto compressed = compressHash(placeholder.hash, 20); + auto clearText = "nix-computed-output:" + + compressed.to_string(Base32, false) + + ":" + std::string { outputName }; + return DownstreamPlaceholder { + hashString(htSHA256, clearText) + }; +} + +} diff -Nru nix-2.8.0/src/libstore/downstream-placeholder.hh nix-2.16.1+dfsg/src/libstore/downstream-placeholder.hh --- nix-2.8.0/src/libstore/downstream-placeholder.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/downstream-placeholder.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,75 @@ +#pragma once +///@file + +#include "hash.hh" +#include "path.hh" + +namespace nix { + +/** + * Downstream Placeholders are opaque and almost certainly unique values + * used to allow derivations to refer to store objects which are yet to + * be built and for we do not yet have store paths for. + * + * They correspond to `DerivedPaths` that are not `DerivedPath::Opaque`, + * except for the cases involving input addressing or fixed outputs + * where we do know a store path for the derivation output in advance. + * + * Unlike `DerivationPath`, however, `DownstreamPlaceholder` is + * purposefully opaque and obfuscated. This is so they are hard to + * create by accident, and so substituting them (once we know what the + * path to store object is) is unlikely to capture other stuff it + * shouldn't. + * + * We use them with `Derivation`: the `render()` method is called to + * render an opaque string which can be used in the derivation, and the + * resolving logic can substitute those strings for store paths when + * resolving `Derivation.inputDrvs` to `BasicDerivation.inputSrcs`. + */ +class DownstreamPlaceholder +{ + /** + * `DownstreamPlaceholder` is just a newtype of `Hash`. + * This its only field. + */ + Hash hash; + + /** + * Newtype constructor + */ + DownstreamPlaceholder(Hash hash) : hash(hash) { } + +public: + /** + * This creates an opaque and almost certainly unique string + * deterministically from the placeholder. + */ + std::string render() const; + + /** + * Create a placeholder for an unknown output of a content-addressed + * derivation. + * + * The derivation itself is known (we have a store path for it), but + * the output doesn't yet have a known store path. + */ + static DownstreamPlaceholder unknownCaOutput( + const StorePath & drvPath, + std::string_view outputName); + + /** + * Create a placehold for the output of an unknown derivation. + * + * The derivation is not yet known because it is a dynamic + * derivaiton --- it is itself an output of another derivation --- + * and we just have (another) placeholder for it. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DownstreamPlaceholder unknownDerivation( + const DownstreamPlaceholder & drvPlaceholder, + std::string_view outputName, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); +}; + +} diff -Nru nix-2.8.0/src/libstore/dummy-store.cc nix-2.16.1+dfsg/src/libstore/dummy-store.cc --- nix-2.8.0/src/libstore/dummy-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/dummy-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -7,6 +7,13 @@ using StoreConfig::StoreConfig; const std::string name() override { return "Dummy Store"; } + + std::string doc() override + { + return + #include "dummy-store.md" + ; + } }; struct DummyStore : public virtual DummyStoreConfig, public virtual Store @@ -32,6 +39,14 @@ callback(nullptr); } + /** + * The dummy store is incapable of *not* trusting! :) + */ + virtual std::optional isTrustedClient() override + { + return Trusted; + } + static std::set uriSchemes() { return {"dummy"}; } @@ -56,6 +71,9 @@ void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override { callback(nullptr); } + + virtual ref getFSAccessor() override + { unsupported("getFSAccessor"); } }; static RegisterStoreImplementation regDummyStore; diff -Nru nix-2.8.0/src/libstore/dummy-store.md nix-2.16.1+dfsg/src/libstore/dummy-store.md --- nix-2.8.0/src/libstore/dummy-store.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/dummy-store.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,13 @@ +R"( + +**Store URL format**: `dummy://` + +This store type represents a store that contains no store paths and +cannot be written to. It's useful when you want to use the Nix +evaluator when no actual Nix store exists, e.g. + +```console +# nix eval --store dummy:// --expr '1 + 2' +``` + +)" diff -Nru nix-2.8.0/src/libstore/export-import.cc nix-2.16.1+dfsg/src/libstore/export-import.cc --- nix-2.8.0/src/libstore/export-import.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/export-import.cc 2023-06-06 10:55:23.000000000 +0000 @@ -16,7 +16,7 @@ //logger->incExpected(doneLabel, sorted.size()); for (auto & path : sorted) { - //Activity act(*logger, lvlInfo, format("exporting path '%s'") % path); + //Activity act(*logger, lvlInfo, "exporting path '%s'", path); sink << 1; exportPath(path, sink); //logger->incProgress(doneLabel); @@ -45,7 +45,7 @@ teeSink << exportMagic << printStorePath(path); - worker_proto::write(*this, teeSink, info->references); + workerProtoWrite(*this, teeSink, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -71,9 +71,9 @@ auto path = parseStorePath(readString(source)); - //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); + //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); - auto references = worker_proto::read(*this, source, Phantom {}); + auto references = WorkerProto::read(*this, source); auto deriver = readString(source); auto narHash = hashString(htSHA256, saved.s); diff -Nru nix-2.8.0/src/libstore/filetransfer.cc nix-2.16.1+dfsg/src/libstore/filetransfer.cc --- nix-2.8.0/src/libstore/filetransfer.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/filetransfer.cc 2023-06-06 10:55:23.000000000 +0000 @@ -33,14 +33,6 @@ static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); -std::string resolveUri(std::string_view uri) -{ - if (uri.compare(0, 8, "channel:") == 0) - return "https://nixos.org/channels/" + std::string(uri.substr(8)) + "/nixexprs.tar.xz"; - else - return std::string(uri); -} - struct curlFileTransfer : public FileTransfer { CURLM * curlm = 0; @@ -96,6 +88,10 @@ {request.uri}, request.parentAct) , callback(std::move(callback)) , finalSink([this](std::string_view data) { + if (errorSink) { + (*errorSink)(data); + } + if (this->request.dataCallback) { auto httpStatus = getHTTPStatus(); @@ -109,6 +105,7 @@ this->result.data.append(data); }) { + requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz"); if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); if (!request.mimeType.empty()) @@ -142,9 +139,9 @@ } template - void fail(const T & e) + void fail(T && e) { - failEx(std::make_exception_ptr(e)); + failEx(std::make_exception_ptr(std::move(e))); } LambdaSink finalSink; @@ -170,8 +167,6 @@ } } - if (errorSink) - (*errorSink)({(char *) contents, realSize}); (*decompressionSink)({(char *) contents, realSize}); return realSize; @@ -190,7 +185,7 @@ { size_t realSize = size * nmemb; std::string line((char *) contents, realSize); - printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); + printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); std::smatch match; if (std::regex_match(line, match, statusLine)) { @@ -214,7 +209,7 @@ long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); if (result.etag == request.expectedETag && httpStatus == 200) { - debug(format("shutting down on 200 HTTP response with expected ETag")); + debug("shutting down on 200 HTTP response with expected ETag"); return 0; } } else if (name == "content-encoding") @@ -308,6 +303,9 @@ curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); + if (settings.downloadSpeed.get() > 0) + curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024)); + if (request.head) curl_easy_setopt(req, CURLOPT_NOBODY, 1); @@ -319,9 +317,8 @@ } if (request.verifyTLS) { - debug("verify TLS: Nix CA file = '%s'", settings.caFile); if (settings.caFile != "") - curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); + curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str()); } else { curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); @@ -410,6 +407,10 @@ err = Misc; } else { // Don't bother retrying on certain cURL errors either + + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (code) { case CURLE_FAILED_INIT: case CURLE_URL_MALFORMAT: @@ -430,6 +431,7 @@ default: // Shut up warnings break; } + #pragma GCC diagnostic pop } attempt++; @@ -443,14 +445,13 @@ : httpStatus != 0 ? FileTransferError(err, std::move(response), - fmt("unable to %s '%s': HTTP error %d ('%s')", - request.verb(), request.uri, httpStatus, statusMsg) - + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) - ) + "unable to %s '%s': HTTP error %d%s", + request.verb(), request.uri, httpStatus, + code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) : FileTransferError(err, std::move(response), - fmt("unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code)); + "unable to %s '%s': %s (%d)", + request.verb(), request.uri, curl_easy_strerror(code), code); /* If this is a transient error, then maybe retry the download after a while. If we're writing to a @@ -471,7 +472,7 @@ fileTransfer.enqueueItem(shared_from_this()); } else - fail(exc); + fail(std::move(exc)); } } }; @@ -693,10 +694,10 @@ #if ENABLE_S3 auto [bucketName, key, params] = parseS3Uri(request.uri); - std::string profile = get(params, "profile").value_or(""); - std::string region = get(params, "region").value_or(Aws::Region::US_EAST_1); - std::string scheme = get(params, "scheme").value_or(""); - std::string endpoint = get(params, "endpoint").value_or(""); + std::string profile = getOr(params, "profile", ""); + std::string region = getOr(params, "region", Aws::Region::US_EAST_1); + std::string scheme = getOr(params, "scheme", ""); + std::string endpoint = getOr(params, "endpoint", ""); S3Helper s3Helper(profile, region, scheme, endpoint); @@ -704,7 +705,7 @@ auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, {}, "S3 object '%s' does not exist", request.uri); + throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri); res.data = std::move(*s3Res.data); callback(std::move(res)); #else @@ -835,7 +836,7 @@ { auto state(_state->lock()); - while (state->data.empty()) { + if (state->data.empty()) { if (state->quit) { if (state->exc) std::rethrow_exception(state->exc); @@ -843,6 +844,8 @@ } state.wait(state->avail); + + if (state->data.empty()) continue; } chunk = std::move(state->data); @@ -872,14 +875,4 @@ err.msg = hf; } -bool isUri(std::string_view s) -{ - if (s.compare(0, 8, "channel:") == 0) return true; - size_t pos = s.find("://"); - if (pos == std::string::npos) return false; - std::string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; -} - - } diff -Nru nix-2.8.0/src/libstore/filetransfer.hh nix-2.16.1+dfsg/src/libstore/filetransfer.hh --- nix-2.8.0/src/libstore/filetransfer.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/filetransfer.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" @@ -87,47 +88,59 @@ { virtual ~FileTransfer() { } - /* Enqueue a data transfer request, returning a future to the result of - the download. The future may throw a FileTransferError - exception. */ + /** + * Enqueue a data transfer request, returning a future to the result of + * the download. The future may throw a FileTransferError + * exception. + */ virtual void enqueueFileTransfer(const FileTransferRequest & request, Callback callback) = 0; std::future enqueueFileTransfer(const FileTransferRequest & request); - /* Synchronously download a file. */ + /** + * Synchronously download a file. + */ FileTransferResult download(const FileTransferRequest & request); - /* Synchronously upload a file. */ + /** + * Synchronously upload a file. + */ FileTransferResult upload(const FileTransferRequest & request); - /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. */ + /** + * Download a file, writing its data to a sink. The sink will be + * invoked on the thread of the caller. + */ void download(FileTransferRequest && request, Sink & sink); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -/* Return a shared FileTransfer object. Using this object is preferred - because it enables connection reuse and HTTP/2 multiplexing. */ +/** + * @return a shared FileTransfer object. + * + * Using this object is preferred because it enables connection reuse + * and HTTP/2 multiplexing. + */ ref getFileTransfer(); -/* Return a new FileTransfer object. */ +/** + * @return a new FileTransfer object + * + * Prefer getFileTransfer() to this; see its docs for why. + */ ref makeFileTransfer(); class FileTransferError : public Error { public: FileTransfer::Error error; - std::optional response; // intentionally optional + /// intentionally optional + std::optional response; template FileTransferError(FileTransfer::Error error, std::optional response, const Args & ... args); }; -bool isUri(std::string_view s); - -/* Resolve deprecated 'channel:' URLs. */ -std::string resolveUri(std::string_view uri); - } diff -Nru nix-2.8.0/src/libstore/fs-accessor.hh nix-2.16.1+dfsg/src/libstore/fs-accessor.hh --- nix-2.8.0/src/libstore/fs-accessor.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/fs-accessor.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,11 +1,14 @@ #pragma once +///@file #include "types.hh" namespace nix { -/* An abstract class for accessing a filesystem-like structure, such - as a (possibly remote) Nix store or the contents of a NAR file. */ +/** + * An abstract class for accessing a filesystem-like structure, such + * as a (possibly remote) Nix store or the contents of a NAR file. + */ class FSAccessor { public: @@ -14,8 +17,17 @@ struct Stat { Type type = tMissing; - uint64_t fileSize = 0; // regular files only + /** + * regular files only + */ + uint64_t fileSize = 0; + /** + * regular files only + */ bool isExecutable = false; // regular files only + /** + * regular files only + */ uint64_t narOffset = 0; // regular files only }; diff -Nru nix-2.8.0/src/libstore/gc.cc nix-2.16.1+dfsg/src/libstore/gc.cc --- nix-2.8.0/src/libstore/gc.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/gc.cc 2023-06-06 10:55:23.000000000 +0000 @@ -34,14 +34,11 @@ createDirs(dirOf(link)); /* Create the new symlink. */ - Path tempLink = (format("%1%.tmp-%2%-%3%") - % link % getpid() % random()).str(); + Path tempLink = fmt("%1%.tmp-%2%-%3%", link, getpid(), random()); createSymlink(target, tempLink); /* Atomically replace the old one. */ - if (rename(tempLink.c_str(), link.c_str()) == -1) - throw SysError("cannot rename '%1%' to '%2%'", - tempLink , link); + renameFile(tempLink, link); } @@ -79,90 +76,106 @@ } -void LocalStore::addTempRoot(const StorePath & path) +void LocalStore::createTempRootsFile() { - auto state(_state.lock()); + auto fdTempRoots(_fdTempRoots.lock()); /* Create the temporary roots file for this process. */ - if (!state->fdTempRoots) { - - while (1) { - if (pathExists(fnTempRoots)) - /* It *must* be stale, since there can be no two - processes with the same pid. */ - unlink(fnTempRoots.c_str()); + if (*fdTempRoots) return; - state->fdTempRoots = openLockFile(fnTempRoots, true); - - debug("acquiring write lock on '%s'", fnTempRoots); - lockFile(state->fdTempRoots.get(), ltWrite, true); + while (1) { + if (pathExists(fnTempRoots)) + /* It *must* be stale, since there can be no two + processes with the same pid. */ + unlink(fnTempRoots.c_str()); + + *fdTempRoots = openLockFile(fnTempRoots, true); + + debug("acquiring write lock on '%s'", fnTempRoots); + lockFile(fdTempRoots->get(), ltWrite, true); + + /* Check whether the garbage collector didn't get in our + way. */ + struct stat st; + if (fstat(fdTempRoots->get(), &st) == -1) + throw SysError("statting '%1%'", fnTempRoots); + if (st.st_size == 0) break; + + /* The garbage collector deleted this file before we could get + a lock. (It won't delete the file after we get a lock.) + Try again. */ + } +} - /* Check whether the garbage collector didn't get in our - way. */ - struct stat st; - if (fstat(state->fdTempRoots.get(), &st) == -1) - throw SysError("statting '%1%'", fnTempRoots); - if (st.st_size == 0) break; - /* The garbage collector deleted this file before we could - get a lock. (It won't delete the file after we get a - lock.) Try again. */ - } +void LocalStore::addTempRoot(const StorePath & path) +{ + createTempRootsFile(); + /* Open/create the global GC lock file. */ + { + auto fdGCLock(_fdGCLock.lock()); + if (!*fdGCLock) + *fdGCLock = openGCLock(); } - if (!state->fdGCLock) - state->fdGCLock = openGCLock(); - restart: - FdLock gcLock(state->fdGCLock.get(), ltRead, false, ""); + /* Try to acquire a shared global GC lock (non-blocking). This + only succeeds if the garbage collector is not currently + running. */ + FdLock gcLock(_fdGCLock.lock()->get(), ltRead, false, ""); if (!gcLock.acquired) { /* We couldn't get a shared global GC lock, so the garbage collector is running. So we have to connect to the garbage collector and inform it about our root. */ - if (!state->fdRootsSocket) { + auto fdRootsSocket(_fdRootsSocket.lock()); + + if (!*fdRootsSocket) { auto socketPath = stateDir.get() + gcSocketPath; debug("connecting to '%s'", socketPath); - state->fdRootsSocket = createUnixDomainSocket(); + *fdRootsSocket = createUnixDomainSocket(); try { - nix::connect(state->fdRootsSocket.get(), socketPath); + nix::connect(fdRootsSocket->get(), socketPath); } catch (SysError & e) { /* The garbage collector may have exited, so we need to restart. */ if (e.errNo == ECONNREFUSED) { debug("GC socket connection refused"); - state->fdRootsSocket.close(); + fdRootsSocket->close(); goto restart; } + throw; } } try { debug("sending GC root '%s'", printStorePath(path)); - writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false); + writeFull(fdRootsSocket->get(), printStorePath(path) + "\n", false); char c; - readFull(state->fdRootsSocket.get(), &c, 1); + readFull(fdRootsSocket->get(), &c, 1); assert(c == '1'); debug("got ack for GC root '%s'", printStorePath(path)); } catch (SysError & e) { /* The garbage collector may have exited, so we need to restart. */ - if (e.errNo == EPIPE) { + if (e.errNo == EPIPE || e.errNo == ECONNRESET) { debug("GC socket disconnected"); - state->fdRootsSocket.close(); + fdRootsSocket->close(); goto restart; } + throw; } catch (EndOfFile & e) { debug("GC socket disconnected"); - state->fdRootsSocket.close(); + fdRootsSocket->close(); goto restart; } } - /* Append the store path to the temporary roots file. */ + /* Record the store path in the temporary roots file so it will be + seen by a future run of the garbage collector. */ auto s = printStorePath(path) + '\0'; - writeFull(state->fdTempRoots.get(), s); + writeFull(_fdTempRoots.lock()->get(), s); } @@ -183,7 +196,7 @@ pid_t pid = std::stoi(i.name); - debug(format("reading temporary root file '%1%'") % path); + debug("reading temporary root file '%1%'", path); AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)); if (!fd) { /* It's okay if the file has disappeared. */ @@ -249,7 +262,7 @@ target = absPath(target, dirOf(path)); if (!pathExists(target)) { if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { - printInfo(format("removing stale link from '%1%' to '%2%'") % path % target); + printInfo("removing stale link from '%1%' to '%2%'", path, target); unlink(path.c_str()); } } else { @@ -358,29 +371,29 @@ while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); - readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); + try { + readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); - auto fdStr = fmt("/proc/%s/fd", ent->d_name); - auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); - if (!fdDir) { - if (errno == ENOENT || errno == EACCES) - continue; - throw SysError("opening %1%", fdStr); - } - struct dirent * fd_ent; - while (errno = 0, fd_ent = readdir(fdDir.get())) { - if (fd_ent->d_name[0] != '.') - readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); - } - if (errno) { - if (errno == ESRCH) - continue; - throw SysError("iterating /proc/%1%/fd", ent->d_name); - } - fdDir.reset(); + auto fdStr = fmt("/proc/%s/fd", ent->d_name); + auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); + if (!fdDir) { + if (errno == ENOENT || errno == EACCES) + continue; + throw SysError("opening %1%", fdStr); + } + struct dirent * fd_ent; + while (errno = 0, fd_ent = readdir(fdDir.get())) { + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); + } + if (errno) { + if (errno == ESRCH) + continue; + throw SysError("iterating /proc/%1%/fd", ent->d_name); + } + fdDir.reset(); - try { auto mapFile = fmt("/proc/%s/maps", ent->d_name); auto mapLines = tokenizeString>(readFile(mapFile), "\n"); for (const auto & line : mapLines) { @@ -506,6 +519,7 @@ Finally cleanup([&]() { debug("GC roots server shutting down"); + fdServer.close(); while (true) { auto item = remove_begin(*connections.lock()); if (!item) break; @@ -619,6 +633,17 @@ Path path = storeDir + "/" + std::string(baseName); Path realPath = realStoreDir + "/" + std::string(baseName); + /* There may be temp directories in the store that are still in use + by another process. We need to be sure that we can acquire an + exclusive lock before deleting them. */ + if (baseName.find("tmp-", 0) == 0) { + AutoCloseFD tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY); + if (tmpDirFd.get() == -1 || !lockFile(tmpDirFd.get(), ltWrite, false)) { + debug("skipping locked tempdir '%s'", realPath); + return; + } + } + printInfo("deleting '%1%'", path); results.paths.insert(path); @@ -837,7 +862,7 @@ continue; } - printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); + printMsg(lvlTalkative, "deleting unused link '%1%'", path); if (unlink(path.c_str()) == -1) throw SysError("deleting '%1%'", path); diff -Nru nix-2.8.0/src/libstore/gc-store.hh nix-2.16.1+dfsg/src/libstore/gc-store.hh --- nix-2.8.0/src/libstore/gc-store.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/gc-store.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" @@ -11,19 +12,20 @@ struct GCOptions { - /* Garbage collector operation: - - - `gcReturnLive': return the set of paths reachable from - (i.e. in the closure of) the roots. - - - `gcReturnDead': return the set of paths not reachable from - the roots. - - - `gcDeleteDead': actually delete the latter set. - - - `gcDeleteSpecific': delete the paths listed in - `pathsToDelete', insofar as they are not reachable. - */ + /** + * Garbage collector operation: + * + * - `gcReturnLive`: return the set of paths reachable from + * (i.e. in the closure of) the roots. + * + * - `gcReturnDead`: return the set of paths not reachable from + * the roots. + * + * - `gcDeleteDead`: actually delete the latter set. + * + * - `gcDeleteSpecific`: delete the paths listed in + * `pathsToDelete`, insofar as they are not reachable. + */ typedef enum { gcReturnLive, gcReturnDead, @@ -33,28 +35,38 @@ GCAction action{gcDeleteDead}; - /* If `ignoreLiveness' is set, then reachability from the roots is - ignored (dangerous!). However, the paths must still be - unreferenced *within* the store (i.e., there can be no other - store paths that depend on them). */ + /** + * If `ignoreLiveness` is set, then reachability from the roots is + * ignored (dangerous!). However, the paths must still be + * unreferenced *within* the store (i.e., there can be no other + * store paths that depend on them). + */ bool ignoreLiveness{false}; - /* For `gcDeleteSpecific', the paths to delete. */ + /** + * For `gcDeleteSpecific`, the paths to delete. + */ StorePathSet pathsToDelete; - /* Stop after at least `maxFreed' bytes have been freed. */ + /** + * Stop after at least `maxFreed` bytes have been freed. + */ uint64_t maxFreed{std::numeric_limits::max()}; }; struct GCResults { - /* Depending on the action, the GC roots, or the paths that would - be or have been deleted. */ + /** + * Depending on the action, the GC roots, or the paths that would + * be or have been deleted. + */ PathSet paths; - /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the - number of bytes that would be or was freed. */ + /** + * For `gcReturnDead`, `gcDeleteDead` and `gcDeleteSpecific`, the + * number of bytes that would be or was freed. + */ uint64_t bytesFreed = 0; }; @@ -63,21 +75,27 @@ { inline static std::string operationName = "Garbage collection"; - /* Add an indirect root, which is merely a symlink to `path' from - /nix/var/nix/gcroots/auto/. `path' is supposed - to be a symlink to a store path. The garbage collector will - automatically remove the indirect root when it finds that - `path' has disappeared. */ + /** + * Add an indirect root, which is merely a symlink to `path` from + * `/nix/var/nix/gcroots/auto/`. `path` is supposed + * to be a symlink to a store path. The garbage collector will + * automatically remove the indirect root when it finds that + * `path` has disappeared. + */ virtual void addIndirectRoot(const Path & path) = 0; - /* Find the roots of the garbage collector. Each root is a pair - (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. If - 'censor' is true, privacy-sensitive information about roots - found in /proc is censored. */ + /** + * Find the roots of the garbage collector. Each root is a pair + * `(link, storepath)` where `link` is the path of the symlink + * outside of the Nix store that point to `storePath`. If + * `censor` is true, privacy-sensitive information about roots + * found in `/proc` is censored. + */ virtual Roots findRoots(bool censor) = 0; - /* Perform a garbage collection. */ + /** + * Perform a garbage collection. + */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; }; diff -Nru nix-2.8.0/src/libstore/globals.cc nix-2.16.1+dfsg/src/libstore/globals.cc --- nix-2.8.0/src/libstore/globals.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/globals.cc 2023-06-06 10:55:23.000000000 +0000 @@ -7,12 +7,23 @@ #include #include +#include #include #include #include #include +#include + +#ifdef __GLIBC__ +#include +#include +#include +#endif + +#include "config-impl.hh" + namespace nix { @@ -30,29 +41,22 @@ Settings::Settings() : nixPrefix(NIX_PREFIX) - , nixStore(canonPath(getEnv("NIX_STORE_DIR").value_or(getEnv("NIX_STORE").value_or(NIX_STORE_DIR)))) - , nixDataDir(canonPath(getEnv("NIX_DATA_DIR").value_or(NIX_DATA_DIR))) - , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) - , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) - , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) + , nixStore(canonPath(getEnvNonEmpty("NIX_STORE_DIR").value_or(getEnvNonEmpty("NIX_STORE").value_or(NIX_STORE_DIR)))) + , nixDataDir(canonPath(getEnvNonEmpty("NIX_DATA_DIR").value_or(NIX_DATA_DIR))) + , nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) + , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) + , nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) , nixUserConfFiles(getUserConfigFiles()) - , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR))) - , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) + , nixBinDir(canonPath(getEnvNonEmpty("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) - , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) + , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; - lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; - caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); - if (caFile == "") { - for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) - if (pathExists(fn)) { - caFile = fn; - break; - } - } + auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); + if (sslOverride != "") + caFile = sslOverride; /* Backwards compatibility. */ auto s = getEnv("NIX_REMOTE_SYSTEMS"); @@ -67,12 +71,13 @@ sandboxPaths = tokenizeString("/bin/sh=" SANDBOX_SHELL); #endif - -/* chroot-like behavior from Apple's sandbox */ + /* chroot-like behavior from Apple's sandbox */ #if __APPLE__ sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib"); allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); #endif + + buildHook = getSelfExe().value_or("nix") + " __build-remote"; } void loadConfFile() @@ -114,7 +119,13 @@ unsigned int Settings::getDefaultCores() { - return std::max(1U, std::thread::hardware_concurrency()); + const unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); + const unsigned int maxCPU = getMaxCPU(); + + if (maxCPU > 0) + return maxCPU; + else + return concurrency; } StringSet Settings::getDefaultSystemFeatures() @@ -125,6 +136,10 @@ StringSet features{"nixos-test", "benchmark", "big-parallel"}; #if __linux__ + features.insert("uid-range"); + #endif + + #if __linux__ if (access("/dev/kvm", R_OK | W_OK) == 0) features.insert("kvm"); #endif @@ -148,30 +163,14 @@ // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist") || - pathExists("/System/Library/LaunchDaemons/com.apple.oahd.plist")) { - if (std::string{SYSTEM} == "x86_64-darwin") - extraPlatforms.insert("aarch64-darwin"); - else if (std::string{SYSTEM} == "aarch64-darwin") - extraPlatforms.insert("x86_64-darwin"); - } + if (std::string{SYSTEM} == "aarch64-darwin" && + runProgram(RunOptions {.program = "arch", .args = {"-arch", "x86_64", "/usr/bin/true"}, .mergeStderrToStdout = true}).first == 0) + extraPlatforms.insert("x86_64-darwin"); #endif return extraPlatforms; } -bool Settings::isExperimentalFeatureEnabled(const ExperimentalFeature & feature) -{ - auto & f = experimentalFeatures.get(); - return std::find(f.begin(), f.end(), feature) != f.end(); -} - -void Settings::requireExperimentalFeature(const ExperimentalFeature & feature) -{ - if (!isExperimentalFeatureEnabled(feature)) - throw MissingExperimentalFeature(feature); -} - bool Settings::isWSL1() { struct utsname utsbuf; @@ -181,6 +180,13 @@ return hasSuffix(utsbuf.release, "-Microsoft"); } +Path Settings::getDefaultSSLCertFile() +{ + for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) + if (pathAccessible(fn)) return fn; + return ""; +} + const std::string nixVersion = PACKAGE_VERSION; NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { @@ -189,18 +195,18 @@ {SandboxMode::smDisabled, false}, }); -template<> void BaseSetting::set(const std::string & str, bool append) +template<> SandboxMode BaseSetting::parse(const std::string & str) const { - if (str == "true") value = smEnabled; - else if (str == "relaxed") value = smRelaxed; - else if (str == "false") value = smDisabled; + if (str == "true") return smEnabled; + else if (str == "relaxed") return smRelaxed; + else if (str == "false") return smDisabled; else throw UsageError("option '%s' has invalid value '%s'", name, str); } -template<> bool BaseSetting::isAppendable() +template<> struct BaseSetting::trait { - return false; -} + static constexpr bool appendable = false; +}; template<> std::string BaseSetting::to_string() const { @@ -216,39 +222,39 @@ .longName = name, .description = "Enable sandboxing.", .category = category, - .handler = {[=]() { override(smEnabled); }} + .handler = {[this]() { override(smEnabled); }} }); args.addFlag({ .longName = "no-" + name, .description = "Disable sandboxing.", .category = category, - .handler = {[=]() { override(smDisabled); }} + .handler = {[this]() { override(smDisabled); }} }); args.addFlag({ .longName = "relaxed-" + name, .description = "Enable sandboxing, but allow builds to disable it.", .category = category, - .handler = {[=]() { override(smRelaxed); }} + .handler = {[this]() { override(smRelaxed); }} }); } -void MaxBuildJobsSetting::set(const std::string & str, bool append) +unsigned int MaxBuildJobsSetting::parse(const std::string & str) const { - if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); + if (str == "auto") return std::max(1U, std::thread::hardware_concurrency()); else { if (auto n = string2Int(str)) - value = *n; + return *n; else throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); } } -void PluginFilesSetting::set(const std::string & str, bool append) +Paths PluginFilesSetting::parse(const std::string & str) const { if (pluginsLoaded) throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); - BaseSetting::set(str, append); + return BaseSetting::parse(str); } @@ -285,4 +291,72 @@ settings.pluginFiles.pluginsLoaded = true; } +static void preloadNSS() +{ + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + static std::once_flag dns_resolve_flag; + + std::call_once(dns_resolve_flag, []() { +#ifdef __GLIBC__ + /* On linux, glibc will run every lookup through the nss layer. + * That means every lookup goes, by default, through nscd, which acts as a local + * cache. + * Because we run builds in a sandbox, we also remove access to nscd otherwise + * lookups would leak into the sandbox. + * + * But now we have a new problem, we need to make sure the nss_dns backend that + * does the dns lookups when nscd is not available is loaded or available. + * + * We can't make it available without leaking nix's environment, so instead we'll + * load the backend, and configure nss so it does not try to run dns lookups + * through nscd. + * + * This is technically only used for builtins:fetch* functions so we only care + * about dns. + * + * All other platforms are unaffected. + */ + if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) + warn("unable to load nss_dns backend"); + // FIXME: get hosts entry from nsswitch.conf. + __nss_configure_lookup("hosts", "files dns"); +#endif + }); +} + +static bool initLibStoreDone = false; + +void assertLibStoreInitialized() { + if (!initLibStoreDone) { + printError("The program must call nix::initNix() before calling any libstore library functions."); + abort(); + }; +} + +void initLibStore() { + + initLibUtil(); + + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); + + loadConfFile(); + + preloadNSS(); + + /* On macOS, don't use the per-session TMPDIR (as set e.g. by + sshd). This breaks build users because they don't have access + to the TMPDIR, in particular in ‘nix-store --serve’. */ +#if __APPLE__ + if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) + unsetenv("TMPDIR"); +#endif + + initLibStoreDone = true; +} + + } diff -Nru nix-2.8.0/src/libstore/globals.hh nix-2.16.1+dfsg/src/libstore/globals.hh --- nix-2.8.0/src/libstore/globals.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/globals.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,9 +1,9 @@ #pragma once +///@file #include "types.hh" #include "config.hh" #include "util.hh" -#include "experimental-features.hh" #include #include @@ -26,7 +26,7 @@ options->addSetting(this); } - void set(const std::string & str, bool append = false) override; + unsigned int parse(const std::string & str) const override; }; struct PluginFilesSetting : public BaseSetting @@ -43,9 +43,17 @@ options->addSetting(this); } - void set(const std::string & str, bool append = false) override; + Paths parse(const std::string & str) const override; }; +const uint32_t maxIdsPerBuild = + #if __linux__ + 1 << 16 + #else + 1 + #endif + ; + class Settings : public Config { unsigned int getDefaultCores(); @@ -56,43 +64,63 @@ bool isWSL1(); + Path getDefaultSSLCertFile(); + public: Settings(); Path nixPrefix; - /* The directory where we store sources and derived files. */ + /** + * The directory where we store sources and derived files. + */ Path nixStore; Path nixDataDir; /* !!! fix */ - /* The directory where we log various operations. */ + /** + * The directory where we log various operations. + */ Path nixLogDir; - /* The directory where state is stored. */ + /** + * The directory where state is stored. + */ Path nixStateDir; - /* The directory where system configuration files are stored. */ + /** + * The directory where system configuration files are stored. + */ Path nixConfDir; - /* A list of user configuration files to load. */ + /** + * A list of user configuration files to load. + */ std::vector nixUserConfFiles; - /* The directory where internal helper programs are stored. */ - Path nixLibexecDir; - - /* The directory where the main programs are stored. */ + /** + * The directory where the main programs are stored. + */ Path nixBinDir; - /* The directory where the man pages are stored. */ + /** + * The directory where the man pages are stored. + */ Path nixManDir; - /* File name of the socket the daemon listens to. */ + /** + * File name of the socket the daemon listens to. + */ Path nixDaemonSocketFile; Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", - "The default Nix store to use."}; + R"( + The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to use for most operations. + See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) + for supported store types and settings. + )"}; Setting keepFailed{this, false, "keep-failed", "Whether to keep temporary directories of failed builds."}; @@ -109,7 +137,9 @@ )", {"build-fallback"}}; - /* Whether to show build log output in real time. */ + /** + * Whether to show build log output in real time. + */ bool verboseBuild = true; Setting logLines{this, 10, "log-lines", @@ -129,6 +159,15 @@ )", {"build-max-jobs"}}; + Setting maxSubstitutionJobs{ + this, 16, "max-substitution-jobs", + R"( + This option defines the maximum number of substitution jobs that Nix + will try to run in parallel. The default is `16`. The minimum value + one can choose is `1` and lower values will be interpreted as `1`. + )", + {"substitution-max-jobs"}}; + Setting buildCores{ this, getDefaultCores(), @@ -145,8 +184,10 @@ )", {"build-cores"}, false}; - /* Read-only mode. Don't copy stuff to the store, don't change - the database. */ + /** + * Read-only mode. Don't copy stuff to the store, don't change + * the database. + */ bool readOnlyMode = false; Setting thisSystem{ @@ -195,8 +236,17 @@ )", {"build-timeout"}}; - PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", - "The path of the helper program that executes builds to remote machines."}; + PathSetting buildHook{this, true, "", "build-hook", + R"( + The path to the helper program that executes remote builds. + + Nix communicates with the build hook over `stdio` using a custom protocol to request builds that cannot be performed directly by the Nix daemon. + The default value is the internal Nix binary that implements remote building. + + > **Important** + > + > Change this setting only if you really know what you’re doing. + )"}; Setting builders{ this, "@" + nixConfDir + "/machines", "builders", @@ -274,10 +324,51 @@ If the build users group is empty, builds will be performed under the uid of the Nix process (that is, the uid of the caller if `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if - `NIX_REMOTE` is `daemon`). Obviously, this should not be used in - multi-user settings with untrusted users. + `NIX_REMOTE` is `daemon`). Obviously, this should not be used + with a nix daemon accessible to untrusted clients. + + Defaults to `nixbld` when running as root, *empty* otherwise. + )", + {}, false}; + + Setting autoAllocateUids{this, false, "auto-allocate-uids", + R"( + Whether to select UIDs for builds automatically, instead of using the + users in `build-users-group`. + + UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. )"}; + Setting startId{this, + #if __linux__ + 0x34000000, + #else + 56930, + #endif + "start-id", + "The first UID and GID to use for dynamic ID allocation."}; + + Setting uidCount{this, + #if __linux__ + maxIdsPerBuild * 128, + #else + 128, + #endif + "id-count", + "The number of UIDs/GIDs to use for dynamic ID allocation."}; + + #if __linux__ + Setting useCgroups{ + this, false, "use-cgroups", + R"( + Whether to execute builds inside cgroups. + This is only supported on Linux. + + Cgroups are required and enabled automatically for derivations + that require the `uid-range` system feature. + )"}; + #endif + Setting impersonateLinux26{this, false, "impersonate-linux-26", "Whether to impersonate a Linux 2.6 machine on newer kernels.", {"build-impersonate-linux-26"}}; @@ -310,11 +401,6 @@ )", {"build-max-log-size"}}; - /* When buildRepeat > 0 and verboseBuild == true, whether to print - repeated builds (i.e. builds other than the first one) to - stderr. Hack to prevent Hydra logs from being polluted. */ - bool printRepeatedBuilds = true; - Setting pollInterval{this, 5, "build-poll-interval", "How often (in seconds) to poll for locks."}; @@ -381,9 +467,6 @@ )", {"env-keep-derivations"}}; - /* Whether to lock the Nix client and worker to the same CPU. */ - bool lockCPU; - Setting sandboxMode{ this, #if __linux__ @@ -430,6 +513,9 @@ for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` will only be mounted in the sandbox if it exists in the host filesystem. + If the source is in the Nix store, then its closure will be added to + the sandbox as well. + Depending on how Nix was built, the default value for this option may be empty or provide `/bin/sh` as a bind-mount of `bash`. )", @@ -438,19 +524,6 @@ Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting buildRepeat{ - this, 0, "repeat", - R"( - How many times to repeat builds to check whether they are - deterministic. The default value is 0. If the value is non-zero, - every build is repeated the specified number of times. If the - contents of any of the runs differs from the previous ones and - `enforce-determinism` is true, the build is rejected and the - resulting store paths are not registered as “valid” in Nix’s - database. - )", - {"build-repeat"}}; - #if __linux__ Setting sandboxShmSize{ this, "50%", "sandbox-dev-shm-size", @@ -514,20 +587,20 @@ configuration file, and cannot be passed at the command line. )"}; - Setting enforceDeterminism{ - this, true, "enforce-determinism", - "Whether to fail if repeated builds produce different output. See `repeat`."}; - Setting trustedPublicKeys{ this, {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, "trusted-public-keys", R"( - A whitespace-separated list of public keys. When paths are copied - from another Nix store (such as a binary cache), they must be - signed with one of these keys. For example: - `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`. + A whitespace-separated list of public keys. + + At least one of the following condition must be met + for Nix to accept copying a store object from another + Nix store (such as a substituter): + + - the store object has been signed using a key in the trusted keys list + - the [`require-sigs`](#conf-require-sigs) option has been set to `false` + - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) )", {"binary-cache-public-keys"}}; @@ -563,9 +636,15 @@ R"( If set to `true` (the default), any non-content-addressed path added or copied to the Nix store (e.g. when substituting from a binary - cache) must have a valid signature, that is, be signed using one of - the keys listed in `trusted-public-keys` or `secret-key-files`. Set - to `false` to disable signature checking. + cache) must have a signature by a trusted key. A trusted key is one + listed in `trusted-public-keys`, or a public key counterpart to a + private key stored in a file listed in `secret-key-files`. + + Set to `false` to disable signature checking and trust all + non-content-addressed paths unconditionally. + + (Content-addressed paths are inherently trustworthy and thus + unaffected by this configuration option.) )"}; Setting extraPlatforms{ @@ -612,17 +691,28 @@ Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of URLs of substituters, separated by whitespace. Substituters - are tried based on their Priority value, which each substituter can set + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to be used as substituters, separated by whitespace. + Substituters are tried based on their Priority value, which each substituter can set independently. Lower value means higher priority. The default is `https://cache.nixos.org`, with a Priority of 40. + + At least one of the following conditions must be met for Nix to use + a substituter: + + - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list + - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list + + In addition, each store path should be trusted as described + in [`trusted-public-keys`](#conf-trusted-public-keys) )", {"binary-caches"}}; Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of URLs of substituters, separated by whitespace. These are + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), + separated by whitespace. These are not used by default, but can be enabled by users of the Nix daemon by specifying `--option substituters urls` on the command line. Unprivileged users are only allowed to pass a subset of the @@ -630,24 +720,6 @@ )", {"trusted-binary-caches"}}; - Setting trustedUsers{ - this, {"root"}, "trusted-users", - R"( - A list of names of users (separated by whitespace) that have - additional rights when connecting to the Nix daemon, such as the - ability to specify additional binary caches, or to import unsigned - NARs. You can also specify groups by prefixing them with `@`; for - instance, `@wheel` means all users in the `wheel` group. The default - is `root`. - - > **Warning** - > - > Adding a user to `trusted-users` is essentially equivalent to - > giving that user root access to the system. For example, the user - > can set `sandbox-paths` and thereby obtain read access to - > directories that are otherwise inacessible to them. - )"}; - Setting ttlNegativeNarInfoCache{ this, 3600, "narinfo-cache-negative-ttl", R"( @@ -670,18 +742,6 @@ mismatch if the build isn't reproducible. )"}; - /* ?Who we trust to use the daemon in safe ways */ - Setting allowedUsers{ - this, {"*"}, "allowed-users", - R"( - A list of names of users (separated by whitespace) that are allowed - to connect to the Nix daemon. As with the `trusted-users` option, - you can specify groups by prefixing them with `@`. Also, you can - allow all users by specifying `*`. The default is `*`. - - Note that trusted users are always allowed to connect. - )"}; - Setting printMissing{this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; @@ -749,6 +809,13 @@ /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`. )"}; + Setting downloadSpeed { + this, 0, "download-speed", + R"( + Specify the maximum transfer rate in kilobytes per second you want + Nix to use for downloads. + )"}; + Setting netrcFile{ this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", R"( @@ -774,8 +841,22 @@ > `.netrc`. )"}; - /* Path to the SSL CA file used */ - Path caFile; + Setting caFile{ + this, getDefaultSSLCertFile(), "ssl-cert-file", + R"( + The path of a file containing CA certificates used to + authenticate `https://` downloads. Nix by default will use + the first of the following files that exists: + + 1. `/etc/ssl/certs/ca-certificates.crt` + 2. `/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt` + + The path can be overridden by the following environment + variables, in order of precedence: + + 1. `NIX_SSL_CERT_FILE` + 2. `SSL_CERT_FILE` + )"}; #if __linux__ Setting filterSyscalls{ @@ -802,7 +883,7 @@ )"}; Setting ignoredAcls{ - this, {"security.selinux", "system.nfs4_acl"}, "ignored-acls", + this, {"security.selinux", "system.nfs4_acl", "security.csm"}, "ignored-acls", R"( A list of ACLs that should be ignored, normally Nix attempts to remove all ACLs from files and directories in the Nix store, but @@ -815,12 +896,11 @@ this, {}, "hashed-mirrors", R"( A list of web servers used by `builtins.fetchurl` to obtain files by - hash. The default is `http://tarballs.nixos.org/`. Given a hash type - *ht* and a base-16 hash *h*, Nix will try to download the file from - *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if - they have disappeared from their original URI. For example, given - the default mirror `http://tarballs.nixos.org/`, when building the - derivation + hash. Given a hash type *ht* and a base-16 hash *h*, Nix will try to + download the file from *hashed-mirror*/*ht*/*h*. This allows files to + be downloaded even if they have disappeared from their original URI. + For example, given an example mirror `http://tarballs.nixos.org/`, + when building the derivation ```nix builtins.fetchurl { @@ -880,13 +960,6 @@ are loaded as plugins (non-recursively). )"}; - Setting> experimentalFeatures{this, {}, "experimental-features", - "Experimental Nix features to enable."}; - - bool isExperimentalFeatureEnabled(const ExperimentalFeature &); - - void requireExperimentalFeature(const ExperimentalFeature &); - Setting narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; @@ -902,14 +975,37 @@ resolves to a different location from that of the build machine. You can enable this setting if you are sure you're not going to do that. )"}; + + Setting useXDGBaseDirectories{ + this, false, "use-xdg-base-directories", + R"( + If set to `true`, Nix will conform to the [XDG Base Directory Specification] for files in `$HOME`. + The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/command-ref/env-common.md). + + [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + + > **Warning** + > This changes the location of some well-known symlinks that Nix creates, which might break tools that rely on the old, non-XDG-conformant locations. + + In particular, the following locations change: + + | Old | New | + |-------------------|--------------------------------| + | `~/.nix-profile` | `$XDG_STATE_HOME/nix/profile` | + | `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` | + | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` | + )" + }; }; // FIXME: don't use a global variable. extern Settings settings; -/* This should be called after settings are initialized, but before - anything else */ +/** + * This should be called after settings are initialized, but before + * anything else + */ void initPlugins(); void loadConfFile(); @@ -919,4 +1015,16 @@ extern const std::string nixVersion; +/** + * NB: This is not sufficient. You need to call initNix() + */ +void initLibStore(); + +/** + * It's important to initialize before doing _anything_, which is why we + * call upon the programmer to handle this correctly. However, we only add + * this in a key locations, so as not to litter the code. + */ +void assertLibStoreInitialized(); + } diff -Nru nix-2.8.0/src/libstore/http-binary-cache-store.cc nix-2.16.1+dfsg/src/libstore/http-binary-cache-store.cc --- nix-2.8.0/src/libstore/http-binary-cache-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/http-binary-cache-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -12,7 +12,14 @@ { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const std::string name() override { return "Http Binary Cache Store"; } + const std::string name() override { return "HTTP Binary Cache Store"; } + + std::string doc() override + { + return + #include "http-binary-cache-store.md" + ; + } }; class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore @@ -56,7 +63,7 @@ void init() override { // FIXME: do this lazily? - if (auto cacheInfo = diskCache->cacheExists(cacheUri)) { + if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { wantMassQuery.setDefault(cacheInfo->wantMassQuery); priority.setDefault(cacheInfo->priority); } else { @@ -161,7 +168,12 @@ void getFile(const std::string & path, Callback> callback) noexcept override { - checkEnabled(); + try { + checkEnabled(); + } catch (...) { + callback.rethrow(); + return; + } auto request(makeRequest(path)); @@ -182,6 +194,18 @@ }}); } + /** + * This isn't actually necessary read only. We support "upsert" now, so we + * have a notion of authentication via HTTP POST/PUT. + * + * For now, we conservatively say we don't know. + * + * \todo try to expose our HTTP authentication status. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } }; static RegisterStoreImplementation regHttpBinaryCacheStore; diff -Nru nix-2.8.0/src/libstore/http-binary-cache-store.md nix-2.16.1+dfsg/src/libstore/http-binary-cache-store.md --- nix-2.8.0/src/libstore/http-binary-cache-store.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/http-binary-cache-store.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `http://...`, `https://...` + +This store allows a binary cache to be accessed via the HTTP +protocol. + +)" diff -Nru nix-2.8.0/src/libstore/legacy-ssh-store.cc nix-2.16.1+dfsg/src/libstore/legacy-ssh-store.cc --- nix-2.8.0/src/libstore/legacy-ssh-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/legacy-ssh-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,3 +1,4 @@ +#include "ssh-store-config.hh" #include "archive.hh" #include "pool.hh" #include "remote-store.hh" @@ -12,17 +13,24 @@ namespace nix { -struct LegacySSHStoreConfig : virtual StoreConfig +struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { - using StoreConfig::StoreConfig; - const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"}; - const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; - const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; - const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + using CommonSSHStoreConfig::CommonSSHStoreConfig; - const std::string name() override { return "Legacy SSH Store"; } + const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", + "Path to the `nix-store` executable on the remote machine."}; + + const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + "Maximum number of concurrent SSH connections."}; + + const std::string name() override { return "SSH Store"; } + + std::string doc() override + { + return + #include "legacy-ssh-store.md" + ; + } }; struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store @@ -51,6 +59,7 @@ LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) + , CommonSSHStoreConfig(params) , LegacySSHStoreConfig(params) , Store(params) , host(host) @@ -134,11 +143,10 @@ /* Hash will be set below. FIXME construct ValidPathInfo at end. */ auto info = std::make_shared(path, Hash::dummy); - PathSet references; auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom {}); + info->references = WorkerProto::read(*this, conn->from); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -148,7 +156,7 @@ throw Error("NAR hash is now mandatory"); info->narHash = Hash::parseAnyPrefixed(s); } - info->ca = parseContentAddressOpt(readString(conn->from)); + info->ca = ContentAddress::parseOpt(readString(conn->from)); info->sigs = readStrings(conn->from); auto s = readString(conn->from); @@ -172,7 +180,7 @@ << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + workerProtoWrite(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize @@ -201,7 +209,7 @@ conn->to << exportMagic << printStorePath(info.path); - worker_proto::write(*this, conn->to, info.references); + workerProtoWrite(*this, conn->to, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -255,8 +263,8 @@ << settings.maxLogSize; if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) conn.to - << settings.buildRepeat - << settings.enforceDeterminism; + << 0 // buildRepeat hasn't worked for ages anyway + << 0; if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) { conn.to << ((int) settings.keepFailed); @@ -279,14 +287,18 @@ conn->to.flush(); - BuildResult status { .path = DerivedPath::Built { .drvPath = drvPath } }; + BuildResult status; status.status = (BuildResult::Status) readInt(conn->from); conn->from >> status.errorMsg; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - status.builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); + auto builtOutputs = WorkerProto::read(*this, conn->from); + for (auto && [output, realisation] : builtOutputs) + status.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); } return status; } @@ -317,7 +329,7 @@ conn->to.flush(); - BuildResult result { .path = DerivedPath::Opaque { StorePath::dummy } }; + BuildResult result; result.status = (BuildResult::Status) readInt(conn->from); if (!result.success()) { @@ -329,6 +341,20 @@ void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } + virtual ref getFSAccessor() override + { unsupported("getFSAccessor"); } + + /** + * The default instance would schedule the work on the client side, but + * for consistency with `buildPaths` and `buildDerivation` it should happen + * on the remote side. + * + * We make this fail for now so we can add implement this properly later + * without it being a breaking change. + */ + void repairPath(const StorePath & path) override + { unsupported("repairPath"); } + void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override @@ -343,10 +369,10 @@ conn->to << cmdQueryClosure << includeOutputs; - worker_proto::write(*this, conn->to, paths); + workerProtoWrite(*this, conn->to, paths); conn->to.flush(); - for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) + for (auto & i : WorkerProto::read(*this, conn->from)) out.insert(i); } @@ -359,10 +385,10 @@ << cmdQueryValidPaths << false // lock << maybeSubstitute; - worker_proto::write(*this, conn->to, paths); + workerProtoWrite(*this, conn->to, paths); conn->to.flush(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } void connect() override @@ -376,6 +402,15 @@ return conn->remoteVersion; } + /** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } + void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override // TODO: Implement diff -Nru nix-2.8.0/src/libstore/legacy-ssh-store.md nix-2.16.1+dfsg/src/libstore/legacy-ssh-store.md --- nix-2.8.0/src/libstore/legacy-ssh-store.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/legacy-ssh-store.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `ssh://[username@]hostname` + +This store type allows limited access to a remote store on another +machine via SSH. + +)" diff -Nru nix-2.8.0/src/libstore/local-binary-cache-store.cc nix-2.16.1+dfsg/src/libstore/local-binary-cache-store.cc --- nix-2.8.0/src/libstore/local-binary-cache-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/local-binary-cache-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -11,6 +11,13 @@ using BinaryCacheStoreConfig::BinaryCacheStoreConfig; const std::string name() override { return "Local Binary Cache Store"; } + + std::string doc() override + { + return + #include "local-binary-cache-store.md" + ; + } }; class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore @@ -57,8 +64,7 @@ AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); - if (rename(tmp.c_str(), path2.c_str())) - throw SysError("renaming '%1%' to '%2%'", tmp, path2); + renameFile(tmp, path2); del.cancel(); } @@ -69,6 +75,7 @@ } catch (SysError & e) { if (e.errNo == ENOENT) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); + throw; } } @@ -88,6 +95,10 @@ return paths; } + std::optional isTrustedClient() override + { + return Trusted; + } }; void LocalBinaryCacheStore::init() @@ -107,7 +118,7 @@ std::set LocalBinaryCacheStore::uriSchemes() { - if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1") + if (getEnv("_NIX_FORCE_HTTP") == "1") return {}; else return {"file"}; diff -Nru nix-2.8.0/src/libstore/local-binary-cache-store.md nix-2.16.1+dfsg/src/libstore/local-binary-cache-store.md --- nix-2.8.0/src/libstore/local-binary-cache-store.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/local-binary-cache-store.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,16 @@ +R"( + +**Store URL format**: `file://`*path* + +This store allows reading and writing a binary cache stored in *path* +in the local filesystem. If *path* does not exist, it will be created. + +For example, the following builds or downloads `nixpkgs#hello` into +the local store and then copies it to the binary cache in +`/tmp/binary-cache`: + +``` +# nix copy --to file:///tmp/binary-cache nixpkgs#hello +``` + +)" diff -Nru nix-2.8.0/src/libstore/local-fs-store.cc nix-2.16.1+dfsg/src/libstore/local-fs-store.cc --- nix-2.8.0/src/libstore/local-fs-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/local-fs-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -87,20 +87,8 @@ const std::string LocalFSStore::drvsLogDir = "drvs"; -std::optional LocalFSStore::getBuildLog(const StorePath & path_) +std::optional LocalFSStore::getBuildLogExact(const StorePath & path) { - auto path = path_; - - if (!path.isDerivation()) { - try { - auto info = queryPathInfo(path); - if (!info->deriver) return std::nullopt; - path = *info->deriver; - } catch (InvalidPath &) { - return std::nullopt; - } - } - auto baseName = path.to_string(); for (int j = 0; j < 2; j++) { diff -Nru nix-2.8.0/src/libstore/local-fs-store.hh nix-2.16.1+dfsg/src/libstore/local-fs-store.hh --- nix-2.8.0/src/libstore/local-fs-store.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/local-fs-store.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "gc-store.hh" @@ -9,20 +10,28 @@ struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; + // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes // it to omit the call to the Setting constructor. Clang works fine // either way. + const PathSetting rootDir{(StoreConfig*) this, true, "", - "root", "directory prefixed to all other paths"}; + "root", + "Directory prefixed to all other paths."}; + const PathSetting stateDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, - "state", "directory where Nix will store state"}; + "state", + "Directory where Nix will store state."}; + const PathSetting logDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, - "log", "directory where Nix will store state"}; + "log", + "directory where Nix will store log files."}; + const PathSetting realStoreDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", - "physical path to the Nix store"}; + "Physical path of the Nix store."}; }; class LocalFSStore : public virtual LocalFSStoreConfig, @@ -39,7 +48,9 @@ void narFromPath(const StorePath & path, Sink & sink) override; ref getFSAccessor() override; - /* Register a permanent GC root. */ + /** + * Register a permanent GC root. + */ Path addPermRoot(const StorePath & storePath, const Path & gcRoot); virtual Path getRealStoreDir() { return realStoreDir; } @@ -50,7 +61,7 @@ return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } - std::optional getBuildLog(const StorePath & path) override; + std::optional getBuildLogExact(const StorePath & path) override; }; diff -Nru nix-2.8.0/src/libstore/local.mk nix-2.16.1+dfsg/src/libstore/local.mk --- nix-2.8.0/src/libstore/local.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -13,14 +13,10 @@ libstore_LDFLAGS += -ldl endif -ifdef HOST_DARWIN -libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb -endif - $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox))) ifeq ($(ENABLE_S3), 1) - libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core + libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp endif ifdef HOST_SOLARIS @@ -39,25 +35,28 @@ -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \ -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \ -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ - -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ -DNIX_BIN_DIR=\"$(bindir)\" \ -DNIX_MAN_DIR=\"$(mandir)\" \ -DLSOF=\"$(lsof)\" +ifeq ($(embedded_sandbox_shell),yes) +libstore_CXXFLAGS += -DSANDBOX_SHELL=\"__embedded_sandbox_shell__\" + +$(d)/build/local-derivation-goal.cc: $(d)/embedded-sandbox-shell.gen.hh + +$(d)/embedded-sandbox-shell.gen.hh: $(sandbox_shell) + $(trace-gen) hexdump -v -e '1/1 "0x%x," "\n"' < $< > $@.tmp + @mv $@.tmp $@ +else ifneq ($(sandbox_shell),) libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" endif +endif $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh $(d)/build.cc: -%.gen.hh: % - @echo 'R"foo(' >> $@.tmp - $(trace-gen) cat $< >> $@.tmp - @echo ')foo"' >> $@.tmp - @mv $@.tmp $@ - clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh $(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) diff -Nru nix-2.8.0/src/libstore/local-store.cc nix-2.16.1+dfsg/src/libstore/local-store.cc --- nix-2.8.0/src/libstore/local-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/local-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -44,6 +44,13 @@ namespace nix { +std::string LocalStoreConfig::doc() +{ + return + #include "local-store.md" + ; +} + struct LocalStore::State::Stmts { /* Some precompiled SQLite statements. */ SQLiteStmt RegisterValidPath; @@ -81,7 +88,7 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) { - const int nixCASchemaVersion = 3; + const int nixCASchemaVersion = 4; int curCASchema = getSchema(schemaPath); if (curCASchema != nixCASchemaVersion) { if (curCASchema > nixCASchemaVersion) { @@ -91,6 +98,7 @@ if (!lockFile(lockFd.get(), ltWrite, false)) { printInfo("waiting for exclusive access to the Nix store for ca drvs..."); + lockFile(lockFd.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks lockFile(lockFd.get(), ltWrite, true); } @@ -143,7 +151,22 @@ )"); txn.commit(); } - writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); + if (curCASchema < 4) { + SQLiteTxn txn(db); + db.exec(R"( + create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths + begin + delete from RealisationsRefs where realisationReference in ( + select id from Realisations where outputPath = old.id + ); + end; + -- used by deletion trigger + create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); + )"); + txn.commit(); + } + + writeFile(schemaPath, fmt("%d", nixCASchemaVersion), 0666, true); lockFile(lockFd.get(), ltRead, true); } } @@ -185,8 +208,6 @@ throw SysError("could not set permissions on '%s' to 755", perUserDir); } - createUser(getUserName(), getuid()); - /* Optionally, create directories and set permissions for a multi-user install. */ if (getuid() == 0 && settings.buildUsersGroup != "") { @@ -266,7 +287,7 @@ else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; openDB(*state, true); - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); } else if (curSchema < nixSchemaVersion) { @@ -284,6 +305,7 @@ if (!lockFile(globalLock.get(), ltWrite, false)) { printInfo("waiting for exclusive access to the Nix store..."); + lockFile(globalLock.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks lockFile(globalLock.get(), ltWrite, true); } @@ -314,14 +336,14 @@ txn.commit(); } - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); lockFile(globalLock.get(), ltRead, true); } else openDB(*state, false); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); } @@ -351,7 +373,7 @@ state->stmts->QueryPathFromHashPart.create(state->db, "select path from ValidPaths where path >= ? limit 1;"); state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths"); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { state->stmts->RegisterRealisedOutput.create(state->db, R"( insert into Realisations (drvPath, outputName, outputPath, signatures) @@ -398,6 +420,13 @@ } +LocalStore::LocalStore(std::string scheme, std::string path, const Params & params) + : LocalStore(params) +{ + throw UnimplementedError("LocalStore"); +} + + AutoCloseFD LocalStore::openGCLock() { Path fnGCLock = stateDir + "/gc.lock"; @@ -424,9 +453,9 @@ } try { - auto state(_state.lock()); - if (state->fdTempRoots) { - state->fdTempRoots = -1; + auto fdTempRoots(_fdTempRoots.lock()); + if (*fdTempRoots) { + *fdTempRoots = -1; unlink(fnTempRoots.c_str()); } } catch (...) { @@ -482,18 +511,18 @@ SQLiteStmt stmt; stmt.create(db, "pragma main.journal_mode;"); if (sqlite3_step(stmt) != SQLITE_ROW) - throwSQLiteError(db, "querying journal mode"); + SQLiteError::throw_(db, "querying journal mode"); prevMode = std::string((const char *) sqlite3_column_text(stmt, 0)); } if (prevMode != mode && sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "setting journal mode"); + SQLiteError::throw_(db, "setting journal mode"); /* Increase the auto-checkpoint interval to 40000 pages. This seems enough to ensure that instantiating the NixOS system derivation is done in a single fsync(). */ if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "setting autocheckpoint interval"); + SQLiteError::throw_(db, "setting autocheckpoint interval"); /* Initialise the database schema, if necessary. */ if (create) { @@ -568,7 +597,10 @@ } -static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) +static void canonicalisePathMetaData_( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) { checkInterrupt(); @@ -615,7 +647,7 @@ However, ignore files that we chown'ed ourselves previously to ensure that we don't fail on hard links within the same build (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ - if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { + if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) throw BuildError("invalid ownership on file '%1%'", path); mode_t mode = st.st_mode & ~S_IFMT; @@ -648,14 +680,17 @@ if (S_ISDIR(st.st_mode)) { DirEntries entries = readDirectory(path); for (auto & i : entries) - canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen); + canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen); } } -void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) { - canonicalisePathMetaData_(path, fromUid, inodesSeen); + canonicalisePathMetaData_(path, uidRange, inodesSeen); /* On platforms that don't have lchown(), the top-level path can't be a symlink, since we can't change its ownership. */ @@ -668,76 +703,26 @@ } -void canonicalisePathMetaData(const Path & path, uid_t fromUid) +void canonicalisePathMetaData(const Path & path, + std::optional> uidRange) { InodesSeen inodesSeen; - canonicalisePathMetaData(path, fromUid, inodesSeen); + canonicalisePathMetaData(path, uidRange, inodesSeen); } -void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv) -{ - assert(drvPath.isDerivation()); - std::string drvName(drvPath.name()); - drvName = drvName.substr(0, drvName.size() - drvExtension.size()); - - auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) - { - auto j = drv.env.find(varName); - if (j == drv.env.end() || parseStorePath(j->second) != actual) - throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", - printStorePath(drvPath), varName, printStorePath(actual)); - }; - - - // Don't need the answer, but do this anyways to assert is proper - // combination. The code below is more general and naturally allows - // combinations that are currently prohibited. - drv.type(); - - std::optional hashesModulo; - for (auto & i : drv.outputs) { - std::visit(overloaded { - [&](const DerivationOutput::InputAddressed & doia) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(*this, drv, true); - } - StorePath recomputed = makeOutputPath(i.first, hashesModulo->hashes.at(i.first), drvName); - if (doia.path != recomputed) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); - envHasRightPath(doia.path, i.first); - }, - [&](const DerivationOutput::CAFixed & dof) { - StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); - envHasRightPath(path, i.first); - }, - [&](const DerivationOutput::CAFloating &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Deferred &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Impure &) { - /* Nothing to check */ - }, - }, i.second.raw()); - } -} - void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info)) registerDrvOutput(info); else - throw Error("cannot register realisation '%s' because it lacks a valid signature", info.outPath.to_string()); + throw Error("cannot register realisation '%s' because it lacks a signature by a trusted key", info.outPath.to_string()); } void LocalStore::registerDrvOutput(const Realisation & info) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); retrySQLite([&]() { auto state(_state.lock()); if (auto oldR = queryRealisation_(*state, info.id)) { @@ -836,7 +821,7 @@ derivations). Note that if this throws an error, then the DB transaction is rolled back, so the path validity registration above is undone. */ - if (checkOutputs) checkDerivationOutputs(info.path, drv); + if (checkOutputs) drv.checkInvariants(*this, info.path); for (auto & i : drv.outputsAndOptPaths(*this)) { /* Floating CA derivations have indeterminate output paths until @@ -904,7 +889,7 @@ if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); + if (s) info->ca = ContentAddress::parseOpt(s); /* Get the references. */ auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); @@ -1026,7 +1011,7 @@ return outputs; }); - if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) return outputs; auto drv = readInvalidDerivation(path); @@ -1094,54 +1079,6 @@ } -// FIXME: move this, it's not specific to LocalStore. -void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) -{ - if (!settings.useSubstitutes) return; - for (auto & sub : getDefaultSubstituters()) { - for (auto & path : paths) { - if (infos.count(path.first)) - // Choose first succeeding substituter. - continue; - - auto subPath(path.first); - - // Recompute store path so that we can use a different store root. - if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); - if (sub->storeDir == storeDir) - assert(subPath == path.first); - if (subPath != path.first) - debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); - } else if (sub->storeDir != storeDir) continue; - - debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); - try { - auto info = sub->queryPathInfo(subPath); - - if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) - continue; - - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - infos.insert_or_assign(path.first, SubstitutablePathInfo{ - info->deriver, - info->references, - narInfo ? narInfo->fileSize : 0, - info->narSize}); - } catch (InvalidPath &) { - } catch (SubstituterDisabled &) { - } catch (Error & e) { - if (settings.tryFallback) - logError(e.info()); - else - throw; - } - } - } -} - - void LocalStore::registerValidPath(const ValidPathInfo & info) { registerValidPaths({{info.path, info}}); @@ -1183,8 +1120,7 @@ for (auto & [_, i] : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, - readInvalidDerivation(i.path)); + readInvalidDerivation(i.path).checkInvariants(*this, i.path); } /* Do a topological sort of the paths. This will throw an @@ -1247,7 +1183,7 @@ RepairFlag repair, CheckSigsFlag checkSigs) { if (checkSigs && pathInfoIsUntrusted(info)) - throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); + throw Error("cannot add path '%s' because it lacks a signature by a trusted key", printStorePath(info.path)); addTempRoot(info.path); @@ -1286,7 +1222,7 @@ printStorePath(info.path), info.narSize, hashResult.second); if (info.ca) { - if (auto foHash = std::get_if(&*info.ca)) { + if (auto foHash = std::get_if(&info.ca->raw)) { auto actualFoHash = hashCAPath( foHash->method, foHash->hash.type, @@ -1299,7 +1235,7 @@ actualFoHash.hash.to_string(Base32, true)); } } - if (auto textHash = std::get_if(&*info.ca)) { + if (auto textHash = std::get_if(&info.ca->raw)) { auto actualTextHash = hashString(htSHA256, readFile(realPath)); if (textHash->hash != actualTextHash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", @@ -1312,7 +1248,7 @@ autoGC(); - canonicalisePathMetaData(realPath, -1); + canonicalisePathMetaData(realPath, {}); optimisePath(realPath, repair); // FIXME: combine with hashPath() @@ -1363,13 +1299,15 @@ std::unique_ptr delTempDir; Path tempPath; + Path tempDir; + AutoCloseFD tempDirFd; if (!inMemory) { /* Drain what we pulled so far, and then keep on pulling */ StringSource dumpSource { dump }; ChainSource bothSource { dumpSource, source }; - auto tempDir = createTempDir(realStoreDir, "add"); + std::tie(tempDir, tempDirFd) = createTempDirInStore(); delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; @@ -1383,7 +1321,19 @@ auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name, references); + ContentAddressWithReferences desc = FixedOutputInfo { + .hash = { + .method = method, + .hash = hash, + }, + .references = { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, + }; + + auto dstPath = makeFixedOutputPathFromCA(name, desc); addTempRoot(dstPath); @@ -1403,7 +1353,7 @@ autoGC(); if (inMemory) { - StringSource dumpSource { dump }; + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ if (method == FileIngestionMethod::Recursive) restorePath(realPath, dumpSource); @@ -1411,8 +1361,7 @@ writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - if (rename(tempPath.c_str(), realPath.c_str())) - throw Error("renaming '%s' to '%s'", tempPath, realPath); + moveFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1424,14 +1373,17 @@ narHash = narSink.finish(); } - canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath + canonicalisePathMetaData(realPath, {}); // FIXME: merge into restorePath optimisePath(realPath, repair); - ValidPathInfo info { dstPath, narHash.first }; + ValidPathInfo info { + *this, + name, + std::move(desc), + narHash.first + }; info.narSize = narHash.second; - info.references = references; - info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1448,7 +1400,10 @@ const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); + auto dstPath = makeTextPath(name, TextInfo { + { .hash = hash }, + references, + }); addTempRoot(dstPath); @@ -1466,7 +1421,7 @@ writeFile(realPath, s); - canonicalisePathMetaData(realPath, -1); + canonicalisePathMetaData(realPath, {}); StringSink sink; dumpString(s, sink); @@ -1489,18 +1444,24 @@ /* Create a temporary directory in the store that won't be - garbage-collected. */ -Path LocalStore::createTempDirInStore() + garbage-collected until the returned FD is closed. */ +std::pair LocalStore::createTempDirInStore() { - Path tmpDir; + Path tmpDirFn; + AutoCloseFD tmpDirFd; + bool lockedByUs = false; do { /* There is a slight possibility that `tmpDir' gets deleted by - the GC between createTempDir() and addTempRoot(), so repeat - until `tmpDir' exists. */ - tmpDir = createTempDir(realStoreDir); - addTempRoot(parseStorePath(tmpDir)); - } while (!pathExists(tmpDir)); - return tmpDir; + the GC between createTempDir() and when we acquire a lock on it. + We'll repeat until 'tmpDir' exists and we've locked it. */ + tmpDirFn = createTempDir(realStoreDir, "tmp"); + tmpDirFd = open(tmpDirFn.c_str(), O_RDONLY | O_DIRECTORY); + if (tmpDirFd.get() < 0) { + continue; + } + lockedByUs = lockFile(tmpDirFd.get(), ltWrite, true); + } while (!pathExists(tmpDirFn) || !lockedByUs); + return {tmpDirFn, std::move(tmpDirFd)}; } @@ -1527,7 +1488,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { - printInfo(format("reading the Nix store...")); + printInfo("reading the Nix store..."); bool errors = false; @@ -1686,6 +1647,11 @@ return PROTOCOL_VERSION; } +std::optional LocalStore::isTrustedClient() +{ + return Trusted; +} + #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) @@ -1789,20 +1755,6 @@ } -void LocalStore::createUser(const std::string & userName, uid_t userId) -{ - for (auto & dir : { - fmt("%s/profiles/per-user/%s", stateDir, userName), - fmt("%s/gcroots/per-user/%s", stateDir, userName) - }) { - createDirs(dir); - if (chmod(dir.c_str(), 0755) == -1) - throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, getgid()) == -1) - throw SysError("changing owner of directory '%s'", dir); - } -} - std::optional> LocalStore::queryRealisationCore_( LocalStore::State & state, const DrvOutput & id) @@ -1923,8 +1875,7 @@ writeFile(tmpFile, compress("bzip2", log)); - if (rename(tmpFile.c_str(), logPath.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath); + renameFile(tmpFile, logPath); } std::optional LocalStore::getVersion() @@ -1932,5 +1883,6 @@ return nixVersion; } +static RegisterStoreImplementation regLocalStore; } // namespace nix diff -Nru nix-2.8.0/src/libstore/local-store.hh nix-2.16.1+dfsg/src/libstore/local-store.hh --- nix-2.8.0/src/libstore/local-store.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/local-store.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "sqlite.hh" @@ -18,10 +19,14 @@ namespace nix { -/* Nix store and database schema version. Version 1 (or 0) was Nix <= - 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. - Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ +/** + * Nix store and database schema version. + * + * Version 1 (or 0) was Nix <= + * 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. + * Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is + * Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. + */ const int nixSchemaVersion = 10; @@ -38,49 +43,52 @@ Setting requireSigs{(StoreConfig*) this, settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; + "require-sigs", + "Whether store paths copied into this store should have a trusted signature."}; const std::string name() override { return "Local Store"; } -}; + std::string doc() override; +}; class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore { private: - /* Lock file used for upgrading. */ + /** + * Lock file used for upgrading. + */ AutoCloseFD globalLock; struct State { - /* The SQLite database object. */ + /** + * The SQLite database object. + */ SQLite db; struct Stmts; std::unique_ptr stmts; - /* The global GC lock */ - AutoCloseFD fdGCLock; - - /* The file to which we write our temporary roots. */ - AutoCloseFD fdTempRoots; - - /* Connection to the garbage collector. */ - AutoCloseFD fdRootsSocket; - - /* The last time we checked whether to do an auto-GC, or an - auto-GC finished. */ + /** + * The last time we checked whether to do an auto-GC, or an + * auto-GC finished. + */ std::chrono::time_point lastGCCheck; - /* Whether auto-GC is running. If so, get gcFuture to wait for - the GC to finish. */ + /** + * Whether auto-GC is running. If so, get gcFuture to wait for + * the GC to finish. + */ bool gcRunning = false; std::shared_future gcFuture; - /* How much disk space was available after the previous - auto-GC. If the current available disk space is below - minFree but not much below availAfterGC, then there is no - point in starting a new GC. */ + /** + * How much disk space was available after the previous + * auto-GC. If the current available disk space is below + * minFree but not much below availAfterGC, then there is no + * point in starting a new GC. + */ uint64_t availAfterGC = std::numeric_limits::max(); std::unique_ptr publicKeys; @@ -103,16 +111,26 @@ public: - // Hack for build-remote.cc. + /** + * Hack for build-remote.cc. + */ PathSet locksHeld; - /* Initialise the local store, upgrading the schema if - necessary. */ + /** + * Initialise the local store, upgrading the schema if + * necessary. + */ LocalStore(const Params & params); + LocalStore(std::string scheme, std::string path, const Params & params); ~LocalStore(); - /* Implementations of abstract store API methods. */ + static std::set uriSchemes() + { return {}; } + + /** + * Implementations of abstract store API methods. + */ std::string getUri() override; @@ -136,9 +154,6 @@ StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) override; - bool pathInfoIsUntrusted(const ValidPathInfo &) override; bool realisationIsUntrusted(const Realisation & ) override; @@ -156,6 +171,27 @@ void addTempRoot(const StorePath & path) override; +private: + + void createTempRootsFile(); + + /** + * The file to which we write our temporary roots. + */ + Sync _fdTempRoots; + + /** + * The global GC lock. + */ + Sync _fdGCLock; + + /** + * Connection to the garbage collector. + */ + Sync _fdRootsSocket; + +public: + void addIndirectRoot(const Path & path) override; private: @@ -170,42 +206,52 @@ void collectGarbage(const GCOptions & options, GCResults & results) override; - /* Optimise the disk space usage of the Nix store by hard-linking - files with the same contents. */ + /** + * Optimise the disk space usage of the Nix store by hard-linking + * files with the same contents. + */ void optimiseStore(OptimiseStats & stats); void optimiseStore() override; - /* Optimise a single store path. Optionally, test the encountered - symlinks for corruption. */ + /** + * Optimise a single store path. Optionally, test the encountered + * symlinks for corruption. + */ void optimisePath(const Path & path, RepairFlag repair); bool verifyStore(bool checkContents, RepairFlag repair) override; - /* Register the validity of a path, i.e., that `path' exists, that - the paths referenced by it exists, and in the case of an output - path of a derivation, that it has been produced by a successful - execution of the derivation (or something equivalent). Also - register the hash of the file system contents of the path. The - hash must be a SHA-256 hash. */ + /** + * Register the validity of a path, i.e., that `path` exists, that + * the paths referenced by it exists, and in the case of an output + * path of a derivation, that it has been produced by a successful + * execution of the derivation (or something equivalent). Also + * register the hash of the file system contents of the path. The + * hash must be a SHA-256 hash. + */ void registerValidPath(const ValidPathInfo & info); void registerValidPaths(const ValidPathInfos & infos); unsigned int getProtocol() override; - void vacuumDB(); + std::optional isTrustedClient() override; - void repairPath(const StorePath & path) override; + void vacuumDB(); void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - /* If free disk space in /nix/store if below minFree, delete - garbage until it exceeds maxFree. */ + /** + * If free disk space in /nix/store if below minFree, delete + * garbage until it exceeds maxFree. + */ void autoGC(bool sync = true); - /* Register the store path 'output' as the output named 'outputName' of - derivation 'deriver'. */ + /** + * Register the store path 'output' as the output named 'outputName' of + * derivation 'deriver'. + */ void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void cacheDrvOutputMapping( @@ -235,7 +281,9 @@ void invalidatePath(State & state, const StorePath & path); - /* Delete a path from the Nix store. */ + /** + * Delete a path from the Nix store. + */ void invalidatePathChecked(const StorePath & path); void verifyPath(const Path & path, const StringSet & store, @@ -256,9 +304,7 @@ void findRuntimeRoots(Roots & roots, bool censor); - Path createTempDirInStore(); - - void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); + std::pair createTempDirInStore(); typedef std::unordered_set InodeHash; @@ -270,13 +316,13 @@ bool isValidPath_(State & state, const StorePath & path); void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); - /* Add signatures to a ValidPathInfo or Realisation using the secret keys - specified by the ‘secret-key-files’ option. */ + /** + * Add signatures to a ValidPathInfo or Realisation using the secret keys + * specified by the ‘secret-key-files’ option. + */ void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - void createUser(const std::string & userName, uid_t userId) override; - // XXX: Make a generic `Store` method FixedOutputHash hashCAPath( const FileIngestionMethod & method, @@ -303,16 +349,30 @@ typedef std::set InodesSeen; -/* "Fix", or canonicalise, the meta-data of the files in a store path - after it has been built. In particular: - - the last modification date on each file is set to 1 (i.e., - 00:00:01 1/1/1970 UTC) - - the permissions are set of 444 or 555 (i.e., read-only with or - without execute permission; setuid bits etc. are cleared) - - the owner and group are set to the Nix user and group, if we're - running as root. */ -void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen); -void canonicalisePathMetaData(const Path & path, uid_t fromUid); +/** + * "Fix", or canonicalise, the meta-data of the files in a store path + * after it has been built. In particular: + * + * - the last modification date on each file is set to 1 (i.e., + * 00:00:01 1/1/1970 UTC) + * + * - the permissions are set of 444 or 555 (i.e., read-only with or + * without execute permission; setuid bits etc. are cleared) + * + * - the owner and group are set to the Nix user and group, if we're + * running as root. + * + * If uidRange is not empty, this function will throw an error if it + * encounters files owned by a user outside of the closed interval + * [uidRange->first, uidRange->second]. + */ +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen); +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange); void canonicaliseTimestampAndPermissions(const Path & path); diff -Nru nix-2.8.0/src/libstore/local-store.md nix-2.16.1+dfsg/src/libstore/local-store.md --- nix-2.8.0/src/libstore/local-store.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/local-store.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,39 @@ +R"( + +**Store URL format**: `local`, *root* + +This store type accesses a Nix store in the local filesystem directly +(i.e. not via the Nix daemon). *root* is an absolute path that is +prefixed to other directories such as the Nix store directory. The +store pseudo-URL `local` denotes a store that uses `/` as its root +directory. + +A store that uses a *root* other than `/` is called a *chroot +store*. With such stores, the store directory is "logically" still +`/nix/store`, so programs stored in them can only be built and +executed by `chroot`-ing into *root*. Chroot stores only support +building and running on Linux when [`mount namespaces`](https://man7.org/linux/man-pages/man7/mount_namespaces.7.html) and [`user namespaces`](https://man7.org/linux/man-pages/man7/user_namespaces.7.html) are +enabled. + +For example, the following uses `/tmp/root` as the chroot environment +to build or download `nixpkgs#hello` and then execute it: + +```console +# nix run --store /tmp/root nixpkgs#hello +Hello, world! +``` + +Here, the "physical" store location is `/tmp/root/nix/store`, and +Nix's store metadata is in `/tmp/root/nix/var/nix/db`. + +It is also possible, but not recommended, to change the "logical" +location of the Nix store from its default of `/nix/store`. This makes +it impossible to use default substituters such as +`https://cache.nixos.org/`, and thus you may have to build everything +locally. Here is an example: + +```console +# nix build --store 'local?store=/tmp/my-nix/store&state=/tmp/my-nix/state&log=/tmp/my-nix/log' nixpkgs#hello +``` + +)" diff -Nru nix-2.8.0/src/libstore/lock.cc nix-2.16.1+dfsg/src/libstore/lock.cc --- nix-2.8.0/src/libstore/lock.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/lock.cc 2023-06-06 10:55:23.000000000 +0000 @@ -2,92 +2,201 @@ #include "globals.hh" #include "pathlocks.hh" -#include #include - -#include -#include +#include namespace nix { -UserLock::UserLock() +struct SimpleUserLock : UserLock { - assert(settings.buildUsersGroup != ""); - createDirs(settings.nixStateDir + "/userpool"); -} + AutoCloseFD fdUserLock; + uid_t uid; + gid_t gid; + std::vector supplementaryGIDs; + + uid_t getUID() override { assert(uid); return uid; } + uid_t getUIDCount() override { return 1; } + gid_t getGID() override { assert(gid); return gid; } + + std::vector getSupplementaryGIDs() override { return supplementaryGIDs; } + + static std::unique_ptr acquire() + { + assert(settings.buildUsersGroup != ""); + createDirs(settings.nixStateDir + "/userpool"); + + /* Get the members of the build-users-group. */ + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); + if (!gr) + throw Error("the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup); + + /* Copy the result of getgrnam. */ + Strings users; + for (char * * p = gr->gr_mem; *p; ++p) { + debug("found build user '%s'", *p); + users.push_back(*p); + } + + if (users.empty()) + throw Error("the build users group '%s' has no members", settings.buildUsersGroup); + + /* Find a user account that isn't currently in use for another + build. */ + for (auto & i : users) { + debug("trying user '%s'", i); + + struct passwd * pw = getpwnam(i.c_str()); + if (!pw) + throw Error("the user '%s' in the group '%s' does not exist", i, settings.buildUsersGroup); + + auto fnUserLock = fmt("%s/userpool/%s", settings.nixStateDir,pw->pw_uid); + + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fd) + throw SysError("opening user lock '%s'", fnUserLock); + + if (lockFile(fd.get(), ltWrite, false)) { + auto lock = std::make_unique(); + + lock->fdUserLock = std::move(fd); + lock->uid = pw->pw_uid; + lock->gid = gr->gr_gid; + + /* Sanity check... */ + if (lock->uid == getuid() || lock->uid == geteuid()) + throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup); + + #if __linux__ + /* Get the list of supplementary groups of this build + user. This is usually either empty or contains a + group such as "kvm". */ + int ngroups = 32; // arbitrary initial guess + std::vector gids; + gids.resize(ngroups); + + int err = getgrouplist( + pw->pw_name, pw->pw_gid, + gids.data(), + &ngroups); + + /* Our initial size of 32 wasn't sufficient, the + correct size has been stored in ngroups, so we try + again. */ + if (err == -1) { + gids.resize(ngroups); + err = getgrouplist( + pw->pw_name, pw->pw_gid, + gids.data(), + &ngroups); + } + + // If it failed once more, then something must be broken. + if (err == -1) + throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name); + + // Finally, trim back the GID list to its real size. + for (auto i = 0; i < ngroups; i++) + if (gids[i] != lock->gid) + lock->supplementaryGIDs.push_back(gids[i]); + #endif -bool UserLock::findFreeUser() { - if (enabled()) return true; + return lock; + } + } - /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - throw Error("the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup); - gid = gr->gr_gid; - - /* Copy the result of getgrnam. */ - Strings users; - for (char * * p = gr->gr_mem; *p; ++p) { - debug("found build user '%1%'", *p); - users.push_back(*p); + return nullptr; } +}; + +struct AutoUserLock : UserLock +{ + AutoCloseFD fdUserLock; + uid_t firstUid = 0; + gid_t firstGid = 0; + uid_t nrIds = 1; + + uid_t getUID() override { assert(firstUid); return firstUid; } + + gid_t getUIDCount() override { return nrIds; } + + gid_t getGID() override { assert(firstGid); return firstGid; } + + std::vector getSupplementaryGIDs() override { return {}; } - if (users.empty()) - throw Error("the build users group '%1%' has no members", - settings.buildUsersGroup); - - /* Find a user account that isn't currently in use for another - build. */ - for (auto & i : users) { - debug("trying user '%1%'", i); - - struct passwd * pw = getpwnam(i.c_str()); - if (!pw) - throw Error("the user '%1%' in the group '%2%' does not exist", - i, settings.buildUsersGroup); - - - fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError("opening user lock '%1%'", fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error("the Nix user should not be a member of '%1%'", - settings.buildUsersGroup); - -#if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); + static std::unique_ptr acquire(uid_t nrIds, bool useUserNamespace) + { + #if !defined(__linux__) + useUserNamespace = false; + #endif - supplementaryGIDs.resize(ngroups); -#endif + experimentalFeatureSettings.require(Xp::AutoAllocateUids); + assert(settings.startId > 0); + assert(settings.uidCount % maxIdsPerBuild == 0); + assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); + assert(nrIds <= maxIdsPerBuild); - isEnabled = true; - return true; + createDirs(settings.nixStateDir + "/userpool2"); + + size_t nrSlots = settings.uidCount / maxIdsPerBuild; + + for (size_t i = 0; i < nrSlots; i++) { + debug("trying user slot '%d'", i); + + createDirs(settings.nixStateDir + "/userpool2"); + + auto fnUserLock = fmt("%s/userpool2/slot-%d", settings.nixStateDir, i); + + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fd) + throw SysError("opening user lock '%s'", fnUserLock); + + if (lockFile(fd.get(), ltWrite, false)) { + + auto firstUid = settings.startId + i * maxIdsPerBuild; + + auto pw = getpwuid(firstUid); + if (pw) + throw Error("auto-allocated UID %d clashes with existing user account '%s'", firstUid, pw->pw_name); + + auto lock = std::make_unique(); + lock->fdUserLock = std::move(fd); + lock->firstUid = firstUid; + if (useUserNamespace) + lock->firstGid = firstUid; + else { + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); + if (!gr) + throw Error("the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup); + lock->firstGid = gr->gr_gid; + } + lock->nrIds = nrIds; + return lock; + } } + + return nullptr; } +}; - return false; +std::unique_ptr acquireUserLock(uid_t nrIds, bool useUserNamespace) +{ + if (settings.autoAllocateUids) + return AutoUserLock::acquire(nrIds, useUserNamespace); + else + return SimpleUserLock::acquire(); } -void UserLock::kill() +bool useBuildUsers() { - killUser(uid); + #if __linux__ + static bool b = (settings.buildUsersGroup != "" || settings.autoAllocateUids) && getuid() == 0; + return b; + #elif __APPLE__ + static bool b = settings.buildUsersGroup != "" && getuid() == 0; + return b; + #else + return false; + #endif } } diff -Nru nix-2.8.0/src/libstore/lock.hh nix-2.16.1+dfsg/src/libstore/lock.hh --- nix-2.8.0/src/libstore/lock.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/lock.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,37 +1,45 @@ #pragma once +///@file -#include "sync.hh" #include "types.hh" -#include "util.hh" + +#include + +#include namespace nix { -class UserLock +struct UserLock { -private: - Path fnUserLock; - AutoCloseFD fdUserLock; - - bool isEnabled = false; - std::string user; - uid_t uid = 0; - gid_t gid = 0; - std::vector supplementaryGIDs; + virtual ~UserLock() { } -public: - UserLock(); + /** + * Get the first and last UID. + */ + std::pair getUIDRange() + { + auto first = getUID(); + return {first, first + getUIDCount() - 1}; + } + + /** + * Get the first UID. + */ + virtual uid_t getUID() = 0; - void kill(); + virtual uid_t getUIDCount() = 0; - std::string getUser() { return user; } - uid_t getUID() { assert(uid); return uid; } - uid_t getGID() { assert(gid); return gid; } - std::vector getSupplementaryGIDs() { return supplementaryGIDs; } + virtual gid_t getGID() = 0; - bool findFreeUser(); + virtual std::vector getSupplementaryGIDs() = 0; +}; - bool enabled() { return isEnabled; } +/** + * Acquire a user lock for a UID range of size `nrIds`. Note that this + * may return nullptr if no user is available. + */ +std::unique_ptr acquireUserLock(uid_t nrIds, bool useUserNamespace); -}; +bool useBuildUsers(); } diff -Nru nix-2.8.0/src/libstore/log-store.cc nix-2.16.1+dfsg/src/libstore/log-store.cc --- nix-2.8.0/src/libstore/log-store.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/log-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,12 @@ +#include "log-store.hh" + +namespace nix { + +std::optional LogStore::getBuildLog(const StorePath & path) { + auto maybePath = getBuildDerivationPath(path); + if (!maybePath) + return std::nullopt; + return getBuildLogExact(maybePath.value()); +} + +} diff -Nru nix-2.8.0/src/libstore/log-store.hh nix-2.16.1+dfsg/src/libstore/log-store.hh --- nix-2.8.0/src/libstore/log-store.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/log-store.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" @@ -9,9 +10,13 @@ { inline static std::string operationName = "Build log storage and retrieval"; - /* Return the build log of the specified store path, if available, - or null otherwise. */ - virtual std::optional getBuildLog(const StorePath & path) = 0; + /** + * Return the build log of the specified store path, if available, + * or null otherwise. + */ + std::optional getBuildLog(const StorePath & path); + + virtual std::optional getBuildLogExact(const StorePath & path) = 0; virtual void addBuildLog(const StorePath & path, std::string_view log) = 0; diff -Nru nix-2.8.0/src/libstore/machines.hh nix-2.16.1+dfsg/src/libstore/machines.hh --- nix-2.8.0/src/libstore/machines.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/machines.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff -Nru nix-2.8.0/src/libstore/make-content-addressed.cc nix-2.16.1+dfsg/src/libstore/make-content-addressed.cc --- nix-2.8.0/src/libstore/make-content-addressed.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/make-content-addressed.cc 2023-06-06 10:55:23.000000000 +0000 @@ -27,18 +27,17 @@ StringMap rewrites; - StorePathSet references; - bool hasSelfReference = false; + StoreReferences refs; for (auto & ref : oldInfo->references) { if (ref == path) - hasSelfReference = true; + refs.self = true; else { auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? if (replacement != ref) rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - references.insert(std::move(replacement)); + refs.others.insert(std::move(replacement)); } } @@ -49,24 +48,28 @@ auto narModuloHash = hashModuloSink.finish().first; - auto dstPath = dstStore.makeFixedOutputPath( - FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference); + ValidPathInfo info { + dstStore, + path.name(), + FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, + }, + .references = std::move(refs), + }, + Hash::dummy, + }; - printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath)); + printInfo("rewriting '%s' to '%s'", pathS, dstStore.printStorePath(info.path)); StringSink sink2; - RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2); + RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), sink2); rsink2(sink.s); rsink2.flush(); - ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) }; - info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); + info.narHash = hashString(htSHA256, sink2.s); info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - }; StringSource source(sink2.s); dstStore.addToStore(info, source); diff -Nru nix-2.8.0/src/libstore/make-content-addressed.hh nix-2.16.1+dfsg/src/libstore/make-content-addressed.hh --- nix-2.8.0/src/libstore/make-content-addressed.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/make-content-addressed.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff -Nru nix-2.8.0/src/libstore/misc.cc nix-2.16.1+dfsg/src/libstore/misc.cc --- nix-2.8.0/src/libstore/misc.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/misc.cc 2023-06-06 10:55:23.000000000 +0000 @@ -83,14 +83,15 @@ } -std::optional getDerivationCA(const BasicDerivation & drv) +const ContentAddress * getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); - if (out != drv.outputs.end()) { - if (const auto * v = std::get_if(&out->second.raw())) - return v->hash; + if (out == drv.outputs.end()) + return nullptr; + if (auto dof = std::get_if(&out->second)) { + return &dof->ca; } - return std::nullopt; + return nullptr; } void Store::queryMissing(const std::vector & targets, @@ -140,7 +141,13 @@ if (drvState_->lock()->done) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); + auto * cap = getDerivationCA(*drv); + querySubstitutablePathInfos({ + { + outPath, + cap ? std::optional { *cap } : std::nullopt, + }, + }, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -185,7 +192,7 @@ knownOutputPaths = false; break; } - if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt)) + if (bfd.outputs.contains(outputName) && !isValidPath(*pathOpt)) invalid.insert(*pathOpt); } if (knownOutputPaths && invalid.empty()) return; @@ -278,11 +285,16 @@ std::set inputRealisations; for (const auto & [inputDrv, outputNames] : drv.inputDrvs) { - auto outputHashes = + const auto outputHashes = staticOutputHashes(store, store.readDerivation(inputDrv)); for (const auto & outputName : outputNames) { + auto outputHash = get(outputHashes, outputName); + if (!outputHash) + throw Error( + "output '%s' of derivation '%s' isn't realised", outputName, + store.printStorePath(inputDrv)); auto thisRealisation = store.queryRealisation( - DrvOutput{outputHashes.at(outputName), outputName}); + DrvOutput{*outputHash, outputName}); if (!thisRealisation) throw Error( "output '%s' of derivation '%s' isn't built", outputName, @@ -296,4 +308,47 @@ return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); } +OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + OutputPathMap outputs; + auto drv = evalStore.readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(store, drv); + auto drvOutputs = drv.outputsAndOptPaths(store); + auto outputNames = std::visit(overloaded { + [&](const OutputsSpec::All &) { + StringSet names; + for (auto & [outputName, _] : drv.outputs) + names.insert(outputName); + return names; + }, + [&](const OutputsSpec::Names & names) { + return static_cast>(names); + }, + }, bfd.outputs.raw()); + for (auto & output : outputNames) { + auto outputHash = get(outputHashes, output); + if (!outputHash) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + store.printStorePath(bfd.drvPath), output); + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + DrvOutput outputId { *outputHash, output }; + auto realisation = store.queryRealisation(outputId); + if (!realisation) + throw MissingRealisation(outputId); + outputs.insert_or_assign(output, realisation->outPath); + } else { + // If ca-derivations isn't enabled, assume that + // the output path is statically known. + auto drvOutput = get(drvOutputs, output); + assert(drvOutput); + assert(drvOutput->second); + outputs.insert_or_assign(output, *drvOutput->second); + } + } + return outputs; +} + } diff -Nru nix-2.8.0/src/libstore/names.hh nix-2.16.1+dfsg/src/libstore/names.hh --- nix-2.8.0/src/libstore/names.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/names.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include diff -Nru nix-2.8.0/src/libstore/nar-accessor.cc nix-2.16.1+dfsg/src/libstore/nar-accessor.cc --- nix-2.8.0/src/libstore/nar-accessor.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/nar-accessor.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,5 @@ #include "nar-accessor.hh" #include "archive.hh" -#include "json.hh" #include #include @@ -75,6 +74,9 @@ createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); } + void closeRegularFile() override + { } + void isExecutable() override { parents.top()->isExecutable = true; @@ -240,42 +242,44 @@ return make_ref(listing, getNarBytes); } -void listNar(JSONPlaceholder & res, ref accessor, - const Path & path, bool recurse) +using nlohmann::json; +json listNar(ref accessor, const Path & path, bool recurse) { auto st = accessor->stat(path); - auto obj = res.object(); + json obj = json::object(); switch (st.type) { case FSAccessor::Type::tRegular: - obj.attr("type", "regular"); - obj.attr("size", st.fileSize); + obj["type"] = "regular"; + obj["size"] = st.fileSize; if (st.isExecutable) - obj.attr("executable", true); + obj["executable"] = true; if (st.narOffset) - obj.attr("narOffset", st.narOffset); + obj["narOffset"] = st.narOffset; break; case FSAccessor::Type::tDirectory: - obj.attr("type", "directory"); + obj["type"] = "directory"; { - auto res2 = obj.object("entries"); + obj["entries"] = json::object(); + json &res2 = obj["entries"]; for (auto & name : accessor->readDirectory(path)) { if (recurse) { - auto res3 = res2.placeholder(name); - listNar(res3, accessor, path + "/" + name, true); + res2[name] = listNar(accessor, path + "/" + name, true); } else - res2.object(name); + res2[name] = json::object(); } } break; case FSAccessor::Type::tSymlink: - obj.attr("type", "symlink"); - obj.attr("target", accessor->readLink(path)); + obj["type"] = "symlink"; + obj["target"] = accessor->readLink(path); break; + case FSAccessor::Type::tMissing: default: throw Error("path '%s' does not exist in NAR", path); } + return obj; } } diff -Nru nix-2.8.0/src/libstore/nar-accessor.hh nix-2.16.1+dfsg/src/libstore/nar-accessor.hh --- nix-2.8.0/src/libstore/nar-accessor.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/nar-accessor.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,34 +1,39 @@ #pragma once +///@file #include +#include #include "fs-accessor.hh" namespace nix { struct Source; -/* Return an object that provides access to the contents of a NAR - file. */ +/** + * Return an object that provides access to the contents of a NAR + * file. + */ ref makeNarAccessor(std::string && nar); ref makeNarAccessor(Source & source); -/* Create a NAR accessor from a NAR listing (in the format produced by - listNar()). The callback getNarBytes(offset, length) is used by the - readFile() method of the accessor to get the contents of files - inside the NAR. */ +/** + * Create a NAR accessor from a NAR listing (in the format produced by + * listNar()). The callback getNarBytes(offset, length) is used by the + * readFile() method of the accessor to get the contents of files + * inside the NAR. + */ typedef std::function GetNarBytes; ref makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); -class JSONPlaceholder; - -/* Write a JSON representation of the contents of a NAR (except file - contents). */ -void listNar(JSONPlaceholder & res, ref accessor, - const Path & path, bool recurse); +/** + * Write a JSON representation of the contents of a NAR (except file + * contents). + */ +nlohmann::json listNar(ref accessor, const Path & path, bool recurse); } diff -Nru nix-2.8.0/src/libstore/nar-info.cc nix-2.16.1+dfsg/src/libstore/nar-info.cc --- nix-2.8.0/src/libstore/nar-info.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/nar-info.cc 2023-06-06 10:55:23.000000000 +0000 @@ -7,15 +7,18 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { - auto corrupt = [&]() { - return Error("NAR info file '%1%' is corrupt", whence); + unsigned line = 1; + + auto corrupt = [&](const char * reason) { + return Error("NAR info file '%1%' is corrupt: %2%", whence, + std::string(reason) + (line > 0 ? " at line " + std::to_string(line) : "")); }; auto parseHashField = [&](const std::string & s) { try { return Hash::parseAnyPrefixed(s); } catch (BadHash &) { - throw corrupt(); + throw corrupt("bad hash"); } }; @@ -26,12 +29,12 @@ while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) throw corrupt(); + if (colon == std::string::npos) throw corrupt("expecting ':'"); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) throw corrupt(); + if (eol == std::string::npos) throw corrupt("expecting '\\n'"); std::string value(s, colon + 2, eol - colon - 2); @@ -47,7 +50,7 @@ fileHash = parseHashField(value); else if (name == "FileSize") { auto n = string2Int(value); - if (!n) throw corrupt(); + if (!n) throw corrupt("invalid FileSize"); fileSize = *n; } else if (name == "NarHash") { @@ -56,12 +59,12 @@ } else if (name == "NarSize") { auto n = string2Int(value); - if (!n) throw corrupt(); + if (!n) throw corrupt("invalid NarSize"); narSize = *n; } else if (name == "References") { auto refs = tokenizeString(value, " "); - if (!references.empty()) throw corrupt(); + if (!references.empty()) throw corrupt("extra References"); for (auto & r : refs) references.insert(StorePath(r)); } @@ -69,22 +72,29 @@ if (value != "unknown-deriver") deriver = StorePath(value); } - else if (name == "System") - system = value; else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (ca) throw corrupt(); + if (ca) throw corrupt("extra CA"); // FIXME: allow blank ca or require skipping field? - ca = parseContentAddressOpt(value); + ca = ContentAddress::parseOpt(value); } pos = eol + 1; + line += 1; } if (compression == "") compression = "bzip2"; - if (!havePath || !haveNarHash || url.empty() || narSize == 0) throw corrupt(); + if (!havePath || !haveNarHash || url.empty() || narSize == 0) { + line = 0; // don't include line information in the error + throw corrupt( + !havePath ? "StorePath missing" : + !haveNarHash ? "NarHash missing" : + url.empty() ? "URL missing" : + narSize == 0 ? "NarSize missing or zero" + : "?"); + } } std::string NarInfo::to_string(const Store & store) const @@ -106,9 +116,6 @@ if (deriver) res += "Deriver: " + std::string(deriver->to_string()) + "\n"; - if (!system.empty()) - res += "System: " + system + "\n"; - for (auto sig : sigs) res += "Sig: " + sig + "\n"; diff -Nru nix-2.8.0/src/libstore/nar-info-disk-cache.cc nix-2.16.1+dfsg/src/libstore/nar-info-disk-cache.cc --- nix-2.8.0/src/libstore/nar-info-disk-cache.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/nar-info-disk-cache.cc 2023-06-06 10:55:23.000000000 +0000 @@ -62,6 +62,9 @@ /* How often to purge expired entries from the cache. */ const int purgeInterval = 24 * 3600; + /* How long to cache binary cache info (i.e. /nix-cache-info) */ + const int cacheInfoTtl = 7 * 24 * 3600; + struct Cache { int id; @@ -81,11 +84,10 @@ Sync _state; - NarInfoDiskCacheImpl() + NarInfoDiskCacheImpl(Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite") { auto state(_state.lock()); - Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; createDirs(dirOf(dbPath)); state->db = SQLite(dbPath); @@ -95,10 +97,10 @@ state->db.exec(schema); state->insertCache.create(state->db, - "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); + "insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?1, ?2, ?3, ?4, ?5) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 returning id;"); state->queryCache.create(state->db, - "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); + "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?"); state->insertNAR.create(state->db, "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " @@ -163,38 +165,71 @@ return i->second; } - void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override +private: + + std::optional queryCacheRaw(State & state, const std::string & uri) { - retrySQLite([&]() { + auto i = state.caches.find(uri); + if (i == state.caches.end()) { + auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl)); + if (!queryCache.next()) + return std::nullopt; + auto cache = Cache { + .id = (int) queryCache.getInt(0), + .storeDir = queryCache.getStr(1), + .wantMassQuery = queryCache.getInt(2) != 0, + .priority = (int) queryCache.getInt(3), + }; + state.caches.emplace(uri, cache); + } + return getCache(state, uri); + } + +public: + int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override + { + return retrySQLite([&]() { auto state(_state.lock()); + SQLiteTxn txn(state->db); + + // To avoid the race, we have to check if maybe someone hasn't yet created + // the cache for this URI in the meantime. + auto cache(queryCacheRaw(*state, uri)); + + if (cache) + return cache->id; + + Cache ret { + .id = -1, // set below + .storeDir = storeDir, + .wantMassQuery = wantMassQuery, + .priority = priority, + }; - // FIXME: race + { + auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)); + assert(r.next()); + ret.id = (int) r.getInt(0); + } + + state->caches[uri] = ret; - state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); - assert(sqlite3_changes(state->db) == 1); - state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority}; + txn.commit(); + return ret.id; }); } - std::optional cacheExists(const std::string & uri) override + std::optional upToDateCacheExists(const std::string & uri) override { return retrySQLite>([&]() -> std::optional { auto state(_state.lock()); - - auto i = state->caches.find(uri); - if (i == state->caches.end()) { - auto queryCache(state->queryCache.use()(uri)); - if (!queryCache.next()) - return std::nullopt; - state->caches.emplace(uri, - Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); - } - - auto & cache(getCache(*state, uri)); - + auto cache(queryCacheRaw(*state, uri)); + if (!cache) + return std::nullopt; return CacheInfo { - .wantMassQuery = cache.wantMassQuery, - .priority = cache.priority + .id = cache->id, + .wantMassQuery = cache->wantMassQuery, + .priority = cache->priority }; }); } @@ -238,7 +273,7 @@ narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = ContentAddress::parseOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); @@ -356,4 +391,9 @@ return cache; } +ref getTestNarInfoDiskCache(Path dbPath) +{ + return make_ref(dbPath); +} + } diff -Nru nix-2.8.0/src/libstore/nar-info-disk-cache.hh nix-2.16.1+dfsg/src/libstore/nar-info-disk-cache.hh --- nix-2.8.0/src/libstore/nar-info-disk-cache.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/nar-info-disk-cache.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" #include "nar-info.hh" @@ -13,16 +14,17 @@ virtual ~NarInfoDiskCache() { } - virtual void createCache(const std::string & uri, const Path & storeDir, + virtual int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; struct CacheInfo { + int id; bool wantMassQuery; int priority; }; - virtual std::optional cacheExists(const std::string & uri) = 0; + virtual std::optional upToDateCacheExists(const std::string & uri) = 0; virtual std::pair> lookupNarInfo( const std::string & uri, const std::string & hashPart) = 0; @@ -41,8 +43,12 @@ const std::string & uri, const DrvOutput & id) = 0; }; -/* Return a singleton cache object that can be used concurrently by - multiple threads. */ +/** + * Return a singleton cache object that can be used concurrently by + * multiple threads. + */ ref getNarInfoDiskCache(); +ref getTestNarInfoDiskCache(Path dbPath); + } diff -Nru nix-2.8.0/src/libstore/nar-info.hh nix-2.16.1+dfsg/src/libstore/nar-info.hh --- nix-2.8.0/src/libstore/nar-info.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/nar-info.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" @@ -14,9 +15,11 @@ std::string compression; std::optional fileHash; uint64_t fileSize = 0; - std::string system; NarInfo() = delete; + NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) + : ValidPathInfo(store, std::move(name), std::move(ca), narHash) + { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff -Nru nix-2.8.0/src/libstore/nix-store.pc.in nix-2.16.1+dfsg/src/libstore/nix-store.pc.in --- nix-2.8.0/src/libstore/nix-store.pc.in 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/nix-store.pc.in 2023-06-06 10:55:23.000000000 +0000 @@ -6,4 +6,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff -Nru nix-2.8.0/src/libstore/optimise-store.cc nix-2.16.1+dfsg/src/libstore/optimise-store.cc --- nix-2.8.0/src/libstore/optimise-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/optimise-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -55,7 +55,7 @@ } if (errno) throw SysError("reading directory '%1%'", linksDir); - printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); + printMsg(lvlTalkative, "loaded %1% hash inodes", inodeHash.size()); return inodeHash; } @@ -73,7 +73,7 @@ checkInterrupt(); if (inodeHash.count(dirent->d_ino)) { - debug(format("'%1%' is already linked") % dirent->d_name); + debug("'%1%' is already linked", dirent->d_name); continue; } @@ -102,7 +102,7 @@ if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) { - debug(format("'%1%' is not allowed to be linked in macOS") % path); + debug("'%1%' is not allowed to be linked in macOS", path); return; } #endif @@ -146,7 +146,7 @@ contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true)); + debug("'%1%' has hash '%2%'", path, hash.to_string(Base32, true)); /* Check if this is a known hash. */ Path linkPath = linksDir + "/" + hash.to_string(Base32, false); @@ -196,11 +196,11 @@ auto stLink = lstat(linkPath); if (st.st_ino == stLink.st_ino) { - debug(format("'%1%' is already linked to '%2%'") % path % linkPath); + debug("'%1%' is already linked to '%2%'", path, linkPath); return; } - printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); + printMsg(lvlTalkative, "linking '%1%' to '%2%'", path, linkPath); /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its @@ -213,8 +213,7 @@ its timestamp back to 0. */ MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : ""); - Path tempLink = (format("%1%/.tmp-link-%2%-%3%") - % realStoreDir % getpid() % random()).str(); + Path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), random()); if (link(linkPath.c_str(), tempLink.c_str()) == -1) { if (errno == EMLINK) { @@ -222,14 +221,16 @@ systems). This is likely to happen with empty files. Just shrug and ignore. */ if (st.st_size) - printInfo(format("'%1%' has maximum number of links") % linkPath); + printInfo("'%1%' has maximum number of links", linkPath); return; } throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); } /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) { + try { + renameFile(tempLink, path); + } catch (SysError & e) { if (unlink(tempLink.c_str()) == -1) printError("unable to unlink '%1%'", tempLink); if (errno == EMLINK) { @@ -240,7 +241,7 @@ debug("'%s' has reached maximum number of links", linkPath); return; } - throw SysError("cannot rename '%1%' to '%2%'", tempLink, path); + throw; } stats.filesLinked++; diff -Nru nix-2.8.0/src/libstore/outputs-spec.cc nix-2.16.1+dfsg/src/libstore/outputs-spec.cc --- nix-2.8.0/src/libstore/outputs-spec.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/outputs-spec.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,195 @@ +#include +#include + +#include "util.hh" +#include "regex-combinators.hh" +#include "outputs-spec.hh" +#include "path-regex.hh" + +namespace nix { + +bool OutputsSpec::contains(const std::string & outputName) const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) { + return true; + }, + [&](const OutputsSpec::Names & outputNames) { + return outputNames.count(outputName) > 0; + }, + }, raw()); +} + +static std::string outputSpecRegexStr = + regex::either( + regex::group(R"(\*)"), + regex::group(regex::list(nameRegexStr))); + +std::optional OutputsSpec::parseOpt(std::string_view s) +{ + static std::regex regex(std::string { outputSpecRegexStr }); + + std::smatch match; + std::string s2 { s }; // until some improves std::regex + if (!std::regex_match(s2, match, regex)) + return std::nullopt; + + if (match[1].matched) + return { OutputsSpec::All {} }; + + if (match[2].matched) + return OutputsSpec::Names { tokenizeString(match[2].str(), ",") }; + + assert(false); +} + + +OutputsSpec OutputsSpec::parse(std::string_view s) +{ + std::optional spec = parseOpt(s); + if (!spec) + throw Error("invalid outputs specifier '%s'", s); + return *spec; +} + + +std::optional> ExtendedOutputsSpec::parseOpt(std::string_view s) +{ + auto found = s.rfind('^'); + + if (found == std::string::npos) + return std::pair { s, ExtendedOutputsSpec::Default {} }; + + auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1)); + if (!specOpt) + return std::nullopt; + return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { *std::move(specOpt) } }; +} + + +std::pair ExtendedOutputsSpec::parse(std::string_view s) +{ + std::optional spec = parseOpt(s); + if (!spec) + throw Error("invalid extended outputs specifier '%s'", s); + return *spec; +} + + +std::string OutputsSpec::to_string() const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) -> std::string { + return "*"; + }, + [&](const OutputsSpec::Names & outputNames) -> std::string { + return concatStringsSep(",", outputNames); + }, + }, raw()); +} + + +std::string ExtendedOutputsSpec::to_string() const +{ + return std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default &) -> std::string { + return ""; + }, + [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> std::string { + return "^" + outputSpec.to_string(); + }, + }, raw()); +} + + +OutputsSpec OutputsSpec::union_(const OutputsSpec & that) const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) -> OutputsSpec { + return OutputsSpec::All { }; + }, + [&](const OutputsSpec::Names & theseNames) -> OutputsSpec { + return std::visit(overloaded { + [&](const OutputsSpec::All &) -> OutputsSpec { + return OutputsSpec::All {}; + }, + [&](const OutputsSpec::Names & thoseNames) -> OutputsSpec { + OutputsSpec::Names ret = theseNames; + ret.insert(thoseNames.begin(), thoseNames.end()); + return ret; + }, + }, that.raw()); + }, + }, raw()); +} + + +bool OutputsSpec::isSubsetOf(const OutputsSpec & that) const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) { + return true; + }, + [&](const OutputsSpec::Names & thoseNames) { + return std::visit(overloaded { + [&](const OutputsSpec::All &) { + return false; + }, + [&](const OutputsSpec::Names & theseNames) { + bool ret = true; + for (auto & o : theseNames) + if (thoseNames.count(o) == 0) + ret = false; + return ret; + }, + }, raw()); + }, + }, that.raw()); +} + +} + +namespace nlohmann { + +using namespace nix; + +OutputsSpec adl_serializer::from_json(const json & json) { + auto names = json.get(); + if (names == StringSet({"*"})) + return OutputsSpec::All {}; + else + return OutputsSpec::Names { std::move(names) }; +} + +void adl_serializer::to_json(json & json, OutputsSpec t) { + std::visit(overloaded { + [&](const OutputsSpec::All &) { + json = std::vector({"*"}); + }, + [&](const OutputsSpec::Names & names) { + json = names; + }, + }, t.raw()); +} + + +ExtendedOutputsSpec adl_serializer::from_json(const json & json) { + if (json.is_null()) + return ExtendedOutputsSpec::Default {}; + else { + return ExtendedOutputsSpec::Explicit { json.get() }; + } +} + +void adl_serializer::to_json(json & json, ExtendedOutputsSpec t) { + std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default &) { + json = nullptr; + }, + [&](const ExtendedOutputsSpec::Explicit & e) { + adl_serializer::to_json(json, e); + }, + }, t.raw()); +} + +} diff -Nru nix-2.8.0/src/libstore/outputs-spec.hh nix-2.16.1+dfsg/src/libstore/outputs-spec.hh --- nix-2.8.0/src/libstore/outputs-spec.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/outputs-spec.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,115 @@ +#pragma once +///@file + +#include +#include +#include +#include + +#include "json-impls.hh" + +namespace nix { + +/** + * A non-empty set of outputs, specified by name + */ +struct OutputNames : std::set { + using std::set::set; + + /* These need to be "inherited manually" */ + + OutputNames(const std::set & s) + : std::set(s) + { assert(!empty()); } + + /** + * Needs to be "inherited manually" + */ + OutputNames(std::set && s) + : std::set(s) + { assert(!empty()); } + + /* This set should always be non-empty, so we delete this + constructor in order make creating empty ones by mistake harder. + */ + OutputNames() = delete; +}; + +/** + * The set of all outputs, without needing to name them explicitly + */ +struct AllOutputs : std::monostate { }; + +typedef std::variant _OutputsSpecRaw; + +struct OutputsSpec : _OutputsSpecRaw { + using Raw = _OutputsSpecRaw; + using Raw::Raw; + + /** + * Force choosing a variant + */ + OutputsSpec() = delete; + + using Names = OutputNames; + using All = AllOutputs; + + inline const Raw & raw() const { + return static_cast(*this); + } + + inline Raw & raw() { + return static_cast(*this); + } + + bool contains(const std::string & output) const; + + /** + * Create a new OutputsSpec which is the union of this and that. + */ + OutputsSpec union_(const OutputsSpec & that) const; + + /** + * Whether this OutputsSpec is a subset of that. + */ + bool isSubsetOf(const OutputsSpec & outputs) const; + + /** + * Parse a string of the form 'output1,...outputN' or '*', returning + * the outputs spec. + */ + static OutputsSpec parse(std::string_view s); + static std::optional parseOpt(std::string_view s); + + std::string to_string() const; +}; + +struct DefaultOutputs : std::monostate { }; + +typedef std::variant _ExtendedOutputsSpecRaw; + +struct ExtendedOutputsSpec : _ExtendedOutputsSpecRaw { + using Raw = _ExtendedOutputsSpecRaw; + using Raw::Raw; + + using Default = DefaultOutputs; + using Explicit = OutputsSpec; + + inline const Raw & raw() const { + return static_cast(*this); + } + + /** + * Parse a string of the form 'prefix^output1,...outputN' or + * 'prefix^*', returning the prefix and the extended outputs spec. + */ + static std::pair parse(std::string_view s); + static std::optional> parseOpt(std::string_view s); + + std::string to_string() const; +}; + +} + +JSON_IMPL(OutputsSpec) +JSON_IMPL(ExtendedOutputsSpec) diff -Nru nix-2.8.0/src/libstore/parsed-derivations.cc nix-2.16.1+dfsg/src/libstore/parsed-derivations.cc --- nix-2.8.0/src/libstore/parsed-derivations.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/parsed-derivations.cc 2023-06-06 10:55:23.000000000 +0000 @@ -2,7 +2,6 @@ #include #include -#include "json.hh" namespace nix { @@ -90,6 +89,7 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const { + // FIXME: cache this? StringSet res; for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) res.insert(i); @@ -125,6 +125,11 @@ return getBoolAttr("allowSubstitutes", true); } +bool ParsedDerivation::useUidRange() const +{ + return getRequiredSystemFeatures().count("uid-range"); +} + static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) @@ -144,16 +149,11 @@ auto e = json.find("exportReferencesGraph"); if (e != json.end() && e->is_object()) { for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(store.parseStorePath(p.get())); - store.pathInfoToJSON(jsonRoot, - store.exportReferences(storePaths, inputPaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh + StorePathSet storePaths; + for (auto & p : *i) + storePaths.insert(store.parseStorePath(p.get())); + json[i.key()] = store.pathInfoToJSON( + store.exportReferences(storePaths, inputPaths), false, true); } } diff -Nru nix-2.8.0/src/libstore/parsed-derivations.hh nix-2.16.1+dfsg/src/libstore/parsed-derivations.hh --- nix-2.8.0/src/libstore/parsed-derivations.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/parsed-derivations.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" #include "store-api.hh" @@ -38,6 +39,8 @@ bool substitutesAllowed() const; + bool useUidRange() const; + std::optional prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths); }; diff -Nru nix-2.8.0/src/libstore/path.cc nix-2.16.1+dfsg/src/libstore/path.cc --- nix-2.8.0/src/libstore/path.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/path.cc 2023-06-06 10:55:23.000000000 +0000 @@ -8,8 +8,10 @@ { if (name.empty()) throw BadStorePath("store path '%s' has an empty name", path); - if (name.size() > 211) - throw BadStorePath("store path '%s' has a name longer than 211 characters", path); + if (name.size() > StorePath::MaxPathLen) + throw BadStorePath("store path '%s' has a name longer than %d characters", + path, StorePath::MaxPathLen); + // See nameRegexStr for the definition for (auto c : name) if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') diff -Nru nix-2.8.0/src/libstore/path.hh nix-2.16.1+dfsg/src/libstore/path.hh --- nix-2.8.0/src/libstore/path.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/path.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,22 +1,34 @@ #pragma once +///@file + +#include -#include "content-address.hh" #include "types.hh" namespace nix { -class Store; struct Hash; +/** + * \ref StorePath "Store path" is the fundamental reference type of Nix. + * A store paths refers to a Store object. + * + * See glossary.html#gloss-store-path for more information on a + * conceptual level. + */ class StorePath { std::string baseName; public: - /* Size of the hash part of store paths, in base-32 characters. */ + /** + * Size of the hash part of store paths, in base-32 characters. + */ constexpr static size_t HashLen = 32; // i.e. 160 bits + constexpr static size_t MaxPathLen = 211; + StorePath() = delete; StorePath(std::string_view baseName); @@ -43,8 +55,9 @@ return baseName != other.baseName; } - /* Check whether a file name ends with the extension for - derivations. */ + /** + * Check whether a file name ends with the extension for derivations. + */ bool isDerivation() const; std::string_view name() const @@ -64,11 +77,11 @@ typedef std::set StorePathSet; typedef std::vector StorePaths; -typedef std::map OutputPathMap; - -typedef std::map> StorePathCAMap; -/* Extension of derivations in the Nix store. */ +/** + * The file extension of \ref Derivation derivations when serialized + * into store objects. + */ const std::string drvExtension = ".drv"; } diff -Nru nix-2.8.0/src/libstore/path-info.cc nix-2.16.1+dfsg/src/libstore/path-info.cc --- nix-2.8.0/src/libstore/path-info.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/path-info.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,8 +1,126 @@ #include "path-info.hh" #include "worker-protocol.hh" +#include "store-api.hh" namespace nix { +std::string ValidPathInfo::fingerprint(const Store & store) const +{ + if (narSize == 0) + throw Error("cannot calculate fingerprint of path '%s' because its size is not known", + store.printStorePath(path)); + return + "1;" + store.printStorePath(path) + ";" + + narHash.to_string(Base32, true) + ";" + + std::to_string(narSize) + ";" + + concatStringsSep(",", store.printStorePathSet(references)); +} + + +void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) +{ + sigs.insert(secretKey.signDetached(fingerprint(store))); +} + +std::optional ValidPathInfo::contentAddressWithReferences() const +{ + if (! ca) + return std::nullopt; + + return std::visit(overloaded { + [&](const TextHash & th) -> ContentAddressWithReferences { + assert(references.count(path) == 0); + return TextInfo { + .hash = th, + .references = references, + }; + }, + [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + return FixedOutputInfo { + .hash = foh, + .references = { + .others = std::move(refs), + .self = hasSelfReference, + }, + }; + }, + }, ca->raw); +} + +bool ValidPathInfo::isContentAddressed(const Store & store) const +{ + auto fullCaOpt = contentAddressWithReferences(); + + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt); + + bool res = caPath == path; + + if (!res) + printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); + + return res; +} + + +size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const +{ + if (isContentAddressed(store)) return maxSigs; + + size_t good = 0; + for (auto & sig : sigs) + if (checkSignature(store, publicKeys, sig)) + good++; + return good; +} + + +bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const +{ + return verifyDetached(fingerprint(store), sig, publicKeys); +} + + +Strings ValidPathInfo::shortRefs() const +{ + Strings refs; + for (auto & r : references) + refs.push_back(std::string(r.to_string())); + return refs; +} + + +ValidPathInfo::ValidPathInfo( + const Store & store, + std::string_view name, + ContentAddressWithReferences && ca, + Hash narHash) + : path(store.makeFixedOutputPathFromCA(name, ca)) + , narHash(narHash) +{ + std::visit(overloaded { + [this](TextInfo && ti) { + this->references = std::move(ti.references); + this->ca = std::move((TextHash &&) ti); + }, + [this](FixedOutputInfo && foi) { + this->references = std::move(foi.references.others); + if (foi.references.self) + this->references.insert(path); + this->ca = std::move((FixedOutputHash &&) foi); + }, + }, std::move(ca).raw); +} + + ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) { return read(source, store, format, store.parseStorePath(readString(source))); @@ -14,16 +132,17 @@ auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = worker_proto::read(store, source, Phantom {}); + info.references = WorkerProto::read(store, source); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; info.sigs = readStrings(source); - info.ca = parseContentAddressOpt(readString(source)); + info.ca = ContentAddress::parseOpt(readString(source)); } return info; } + void ValidPathInfo::write( Sink & sink, const Store & store, @@ -34,7 +153,7 @@ sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); - worker_proto::write(store, sink, references); + workerProtoWrite(store, sink, references); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff -Nru nix-2.8.0/src/libstore/path-info.hh nix-2.16.1+dfsg/src/libstore/path-info.hh --- nix-2.8.0/src/libstore/path-info.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/path-info.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "crypto.hh" #include "path.hh" @@ -18,8 +19,14 @@ { std::optional deriver; StorePathSet references; - uint64_t downloadSize; /* 0 = unknown or inapplicable */ - uint64_t narSize; /* 0 = unknown */ + /** + * 0 = unknown or inapplicable + */ + uint64_t downloadSize; + /** + * 0 = unknown + */ + uint64_t narSize; }; typedef std::map SubstitutablePathInfos; @@ -29,35 +36,40 @@ { StorePath path; std::optional deriver; - // TODO document this + /** + * \todo document this + */ Hash narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only - /* Whether the path is ultimately trusted, that is, it's a - derivation output that was built locally. */ + /** + * Whether the path is ultimately trusted, that is, it's a + * derivation output that was built locally. + */ bool ultimate = false; StringSet sigs; // note: not necessarily verified - /* If non-empty, an assertion that the path is content-addressed, - i.e., that the store path is computed from a cryptographic hash - of the contents of the path, plus some other bits of data like - the "name" part of the path. Such a path doesn't need - signatures, since we don't have to trust anybody's claim that - the path is the output of a particular derivation. (In the - extensional store model, we have to trust that the *contents* - of an output path of a derivation were actually produced by - that derivation. In the intensional model, we have to trust - that a particular output path was produced by a derivation; the - path then implies the contents.) - - Ideally, the content-addressability assertion would just be a Boolean, - and the store path would be computed from the name component, ‘narHash’ - and ‘references’. However, we support many types of content addresses. - */ + /** + * If non-empty, an assertion that the path is content-addressed, + * i.e., that the store path is computed from a cryptographic hash + * of the contents of the path, plus some other bits of data like + * the "name" part of the path. Such a path doesn't need + * signatures, since we don't have to trust anybody's claim that + * the path is the output of a particular derivation. (In the + * extensional store model, we have to trust that the *contents* + * of an output path of a derivation were actually produced by + * that derivation. In the intensional model, we have to trust + * that a particular output path was produced by a derivation; the + * path then implies the contents.) + * + * Ideally, the content-addressability assertion would just be a Boolean, + * and the store path would be computed from the name component, ‘narHash’ + * and ‘references’. However, we support many types of content addresses. + */ std::optional ca; bool operator == (const ValidPathInfo & i) const @@ -68,33 +80,42 @@ && references == i.references; } - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ + /** + * Return a fingerprint of the store path to be used in binary + * cache signatures. It contains the store path, the base-32 + * SHA-256 hash of the NAR serialisation of the path, the size of + * the NAR, and the sorted references. The size field is strictly + * speaking superfluous, but might prevent endless/excessive data + * attacks. + */ std::string fingerprint(const Store & store) const; void sign(const Store & store, const SecretKey & secretKey); - /* Return true iff the path is verifiably content-addressed. */ + /** + * @return The `ContentAddressWithReferences` that determines the + * store path for a content-addressed store object, `std::nullopt` + * for an input-addressed store object. + */ + std::optional contentAddressWithReferences() const; + + /** + * @return true iff the path is verifiably content-addressed. + */ bool isContentAddressed(const Store & store) const; - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf() const; - void insertReferencePossiblyToSelf(StorePath && ref); - void setReferencesPossiblyToSelf(StorePathSet && refs); - static const size_t maxSigs = std::numeric_limits::max(); - /* Return the number of signatures on this .narinfo that were - produced by one of the specified keys, or maxSigs if the path - is content-addressed. */ + /** + * Return the number of signatures on this .narinfo that were + * produced by one of the specified keys, or maxSigs if the path + * is content-addressed. + */ size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; - /* Verify a single signature. */ + /** + * Verify a single signature. + */ bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; Strings shortRefs() const; @@ -104,6 +125,9 @@ ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(const Store & store, + std::string_view name, ContentAddressWithReferences && ca, Hash narHash); + virtual ~ValidPathInfo() { } static ValidPathInfo read(Source & source, const Store & store, unsigned int format); diff -Nru nix-2.8.0/src/libstore/pathlocks.cc nix-2.16.1+dfsg/src/libstore/pathlocks.cc --- nix-2.8.0/src/libstore/pathlocks.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/pathlocks.cc 2023-06-06 10:55:23.000000000 +0000 @@ -96,7 +96,7 @@ checkInterrupt(); Path lockPath = path + ".lock"; - debug(format("locking path '%1%'") % path); + debug("locking path '%1%'", path); AutoCloseFD fd; @@ -118,7 +118,7 @@ } } - debug(format("lock acquired on '%1%'") % lockPath); + debug("lock acquired on '%1%'", lockPath); /* Check that the lock file hasn't become stale (i.e., hasn't been unlinked). */ @@ -130,7 +130,7 @@ a lock on a deleted file. This means that other processes may create and acquire a lock on `lockPath', and proceed. So we must retry. */ - debug(format("open lock file '%1%' has become stale") % lockPath); + debug("open lock file '%1%' has become stale", lockPath); else break; } @@ -163,7 +163,7 @@ "error (ignored): cannot close lock file on '%1%'", i.second); - debug(format("lock released on '%1%'") % i.second); + debug("lock released on '%1%'", i.second); } fds.clear(); diff -Nru nix-2.8.0/src/libstore/pathlocks.hh nix-2.16.1+dfsg/src/libstore/pathlocks.hh --- nix-2.8.0/src/libstore/pathlocks.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/pathlocks.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,15 +1,20 @@ #pragma once +///@file #include "util.hh" namespace nix { -/* Open (possibly create) a lock file and return the file descriptor. - -1 is returned if create is false and the lock could not be opened - because it doesn't exist. Any other error throws an exception. */ +/** + * Open (possibly create) a lock file and return the file descriptor. + * -1 is returned if create is false and the lock could not be opened + * because it doesn't exist. Any other error throws an exception. + */ AutoCloseFD openLockFile(const Path & path, bool create); -/* Delete an open lock file. */ +/** + * Delete an open lock file. + */ void deleteLockFile(const Path & path, int fd); enum LockType { ltRead, ltWrite, ltNone }; diff -Nru nix-2.8.0/src/libstore/path-regex.hh nix-2.16.1+dfsg/src/libstore/path-regex.hh --- nix-2.8.0/src/libstore/path-regex.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/path-regex.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,8 @@ +#pragma once +///@file + +namespace nix { + +static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-\._\?=]+)"; + +} diff -Nru nix-2.8.0/src/libstore/path-with-outputs.cc nix-2.16.1+dfsg/src/libstore/path-with-outputs.cc --- nix-2.8.0/src/libstore/path-with-outputs.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/path-with-outputs.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,8 @@ #include "path-with-outputs.hh" #include "store-api.hh" +#include + namespace nix { std::string StorePathWithOutputs::to_string(const Store & store) const @@ -13,10 +15,14 @@ DerivedPath StorePathWithOutputs::toDerivedPath() const { - if (!outputs.empty() || path.isDerivation()) - return DerivedPath::Built { path, outputs }; - else + if (!outputs.empty()) { + return DerivedPath::Built { path, OutputsSpec::Names { outputs } }; + } else if (path.isDerivation()) { + assert(outputs.empty()); + return DerivedPath::Built { path, OutputsSpec::All { } }; + } else { return DerivedPath::Opaque { path }; + } } @@ -39,7 +45,18 @@ return StorePathWithOutputs { bo.path }; }, [&](const DerivedPath::Built & bfd) -> std::variant { - return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; + return StorePathWithOutputs { + .path = bfd.drvPath, + // Use legacy encoding of wildcard as empty set + .outputs = std::visit(overloaded { + [&](const OutputsSpec::All &) -> StringSet { + return {}; + }, + [&](const OutputsSpec::Names & outputs) { + return static_cast(outputs); + }, + }, bfd.outputs.raw()), + }; }, }, p.raw()); } @@ -50,8 +67,8 @@ size_t n = s.find("!"); return n == s.npos ? std::make_pair(s, std::set()) - : std::make_pair(((std::string_view) s).substr(0, n), - tokenizeString>(((std::string_view) s).substr(n + 1), ",")); + : std::make_pair(s.substr(0, n), + tokenizeString>(s.substr(n + 1), ",")); } diff -Nru nix-2.8.0/src/libstore/path-with-outputs.hh nix-2.16.1+dfsg/src/libstore/path-with-outputs.hh --- nix-2.8.0/src/libstore/path-with-outputs.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/path-with-outputs.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,12 +1,19 @@ #pragma once - -#include +///@file #include "path.hh" #include "derived-path.hh" namespace nix { +/** + * This is a deprecated old type just for use by the old CLI, and older + * versions of the RPC protocols. In new code don't use it; you want + * `DerivedPath` instead. + * + * `DerivedPath` is better because it handles more cases, and does so more + * explicitly without devious punning tricks. + */ struct StorePathWithOutputs { StorePath path; @@ -25,9 +32,11 @@ class Store; -/* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path - and the outputs. */ +/** + * Split a string specifying a derivation and a set of outputs + * (/nix/store/hash-foo!out1,out2,...) into the derivation path + * and the outputs. + */ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); diff -Nru nix-2.8.0/src/libstore/profiles.cc nix-2.16.1+dfsg/src/libstore/profiles.cc --- nix-2.8.0/src/libstore/profiles.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/profiles.cc 2023-06-06 10:55:23.000000000 +0000 @@ -64,7 +64,7 @@ static void makeName(const Path & profile, GenerationNumber num, Path & outLink) { - Path prefix = (format("%1%-%2%") % profile % num).str(); + Path prefix = fmt("%1%-%2%", profile, num); outLink = prefix + "-link"; } @@ -269,7 +269,7 @@ void lockProfile(PathLocks & lock, const Path & profile) { - lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str()); + lock.lockPaths({profile}, fmt("waiting for lock on profile '%1%'", profile)); lock.setDeletion(true); } @@ -280,16 +280,35 @@ } +Path profilesDir() +{ + auto profileRoot = + (getuid() == 0) + ? rootProfilesDir() + : createNixStateDir() + "/profiles"; + createDirs(profileRoot); + return profileRoot; +} + +Path rootProfilesDir() +{ + return settings.nixStateDir + "/profiles/per-user/root"; +} + + Path getDefaultProfile() { - Path profileLink = getHome() + "/.nix-profile"; + Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile"; try { + auto profile = profilesDir() + "/profile"; if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); + replaceSymlink(profile, profileLink); + } + // Backwards compatibiliy measure: Make root's profile available as + // `.../default` as it's what NixOS and most of the init scripts expect + Path globalProfileLink = settings.nixStateDir + "/profiles/default"; + if (getuid() == 0 && !pathExists(globalProfileLink)) { + replaceSymlink(profile, globalProfileLink); } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { @@ -297,5 +316,14 @@ } } +Path defaultChannelsDir() +{ + return profilesDir() + "/channels"; +} + +Path rootChannelsDir() +{ + return rootProfilesDir() + "/channels"; +} } diff -Nru nix-2.8.0/src/libstore/profiles.hh nix-2.16.1+dfsg/src/libstore/profiles.hh --- nix-2.8.0/src/libstore/profiles.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/profiles.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,7 @@ #pragma once +///@file -#include "types.hh" + #include "types.hh" #include "pathlocks.hh" #include @@ -23,9 +24,11 @@ typedef std::list Generations; -/* Returns the list of currently present generations for the specified - profile, sorted by generation number. Also returns the number of - the current generation. */ +/** + * Returns the list of currently present generations for the specified + * profile, sorted by generation number. Also returns the number of + * the current generation. + */ std::pair> findGenerations(Path profile); class LocalFSStore; @@ -46,30 +49,60 @@ void switchLink(Path link, Path target); -/* Roll back a profile to the specified generation, or to the most - recent one older than the current. */ +/** + * Roll back a profile to the specified generation, or to the most + * recent one older than the current. + */ void switchGeneration( const Path & profile, std::optional dstGen, bool dryRun); -/* Ensure exclusive access to a profile. Any command that modifies - the profile first acquires this lock. */ +/** + * Ensure exclusive access to a profile. Any command that modifies + * the profile first acquires this lock. + */ void lockProfile(PathLocks & lock, const Path & profile); -/* Optimistic locking is used by long-running operations like `nix-env - -i'. Instead of acquiring the exclusive lock for the entire - duration of the operation, we just perform the operation - optimistically (without an exclusive lock), and check at the end - whether the profile changed while we were busy (i.e., the symlink - target changed). If so, the operation is restarted. Restarting is - generally cheap, since the build results are still in the Nix - store. Most of the time, only the user environment has to be - rebuilt. */ +/** + * Optimistic locking is used by long-running operations like `nix-env + * -i'. Instead of acquiring the exclusive lock for the entire + * duration of the operation, we just perform the operation + * optimistically (without an exclusive lock), and check at the end + * whether the profile changed while we were busy (i.e., the symlink + * target changed). If so, the operation is restarted. Restarting is + * generally cheap, since the build results are still in the Nix + * store. Most of the time, only the user environment has to be + * rebuilt. + */ std::string optimisticLockProfile(const Path & profile); -/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create - it. */ +/** + * Create and return the path to a directory suitable for storing the user’s + * profiles. + */ +Path profilesDir(); + +/** + * Return the path to the profile directory for root (but don't try creating it) + */ +Path rootProfilesDir(); + +/** + * Create and return the path to the file used for storing the users's channels + */ +Path defaultChannelsDir(); + +/** + * Return the path to the channel directory for root (but don't try creating it) + */ +Path rootChannelsDir(); + +/** + * Resolve the default profile (~/.nix-profile by default, + * $XDG_STATE_HOME/nix/profile if XDG Base Directory Support is enabled), + * and create if doesn't exist + */ Path getDefaultProfile(); } diff -Nru nix-2.8.0/src/libstore/realisation.cc nix-2.16.1+dfsg/src/libstore/realisation.cc --- nix-2.8.0/src/libstore/realisation.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/realisation.cc 2023-06-06 10:55:23.000000000 +0000 @@ -136,6 +136,19 @@ return good; } + +SingleDrvOutputs filterDrvOutputs(const OutputsSpec& wanted, SingleDrvOutputs&& outputs) +{ + SingleDrvOutputs ret = std::move(outputs); + for (auto it = ret.begin(); it != ret.end(); ) { + if (!wanted.contains(it->first)) + it = ret.erase(it); + else + ++it; + } + return ret; +} + StorePath RealisedPath::path() const { return std::visit([](auto && arg) { return arg.getPath(); }, raw); } diff -Nru nix-2.8.0/src/libstore/realisation.hh nix-2.16.1+dfsg/src/libstore/realisation.hh --- nix-2.8.0/src/libstore/realisation.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/realisation.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,5 +1,9 @@ #pragma once +///@file +#include + +#include "hash.hh" #include "path.hh" #include #include "comparator.hh" @@ -7,9 +11,28 @@ namespace nix { +class Store; +struct OutputsSpec; + +/** + * A general `Realisation` key. + * + * This is similar to a `DerivedPath::Opaque`, but the derivation is + * identified by its "hash modulo" instead of by its store path. + */ struct DrvOutput { - // The hash modulo of the derivation + /** + * The hash modulo of the derivation. + * + * Computed from the derivation itself for most types of + * derivations, but computed from the (fixed) content address of the + * output for fixed-output derivations. + */ Hash drvHash; + + /** + * The name of the output. + */ std::string outputName; std::string to_string() const; @@ -54,8 +77,31 @@ GENERATE_CMP(Realisation, me->id, me->outPath); }; +/** + * Collection type for a single derivation's outputs' `Realisation`s. + * + * Since these are the outputs of a single derivation, we know the + * output names are unique so we can use them as the map key. + */ +typedef std::map SingleDrvOutputs; + +/** + * Collection type for multiple derivations' outputs' `Realisation`s. + * + * `DrvOutput` is used because in general the derivations are not all + * the same, so we need to identify firstly which derivation, and + * secondly which output of that derivation. + */ typedef std::map DrvOutputs; +/** + * Filter a SingleDrvOutputs to include only specific output names + * + * Moves the `outputs` input. + */ +SingleDrvOutputs filterDrvOutputs(const OutputsSpec&, SingleDrvOutputs&&); + + struct OpaquePath { StorePath path; @@ -93,4 +139,14 @@ GENERATE_CMP(RealisedPath, me->raw); }; +class MissingRealisation : public Error +{ +public: + MissingRealisation(DrvOutput & outputId) + : Error( "cannot operate on an output of the " + "unbuilt derivation '%s'", + outputId.to_string()) + {} +}; + } diff -Nru nix-2.8.0/src/libstore/references.cc nix-2.16.1+dfsg/src/libstore/references.cc --- nix-2.8.0/src/libstore/references.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/references.cc 2023-06-06 10:55:23.000000000 +0000 @@ -39,8 +39,7 @@ if (!match) continue; std::string ref(s.substr(i, refLength)); if (hashes.erase(ref)) { - debug(format("found reference to '%1%' at offset '%2%'") - % ref % i); + debug("found reference to '%1%' at offset '%2%'", ref, i); seen.insert(ref); } ++i; @@ -67,20 +66,12 @@ } -std::pair scanForReferences( - const std::string & path, - const StorePathSet & refs) -{ - HashSink hashSink { htSHA256 }; - auto found = scanForReferences(hashSink, path, refs); - auto hash = hashSink.finish(); - return std::pair(found, hash); -} +PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map && backMap) + : RefScanSink(std::move(hashes)) + , backMap(std::move(backMap)) +{ } -StorePathSet scanForReferences( - Sink & toTee, - const Path & path, - const StorePathSet & refs) +PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) { StringSet hashes; std::map backMap; @@ -92,14 +83,14 @@ hashes.insert(hashPart); } - /* Look for the hashes in the NAR dump of the path. */ - RefScanSink refsSink(std::move(hashes)); - TeeSink sink { refsSink, toTee }; - dumpPath(path, sink); + return PathRefScanSink(std::move(hashes), std::move(backMap)); +} +StorePathSet PathRefScanSink::getResultPaths() +{ /* Map the hashes found back to their store paths. */ StorePathSet found; - for (auto & i : refsSink.getResult()) { + for (auto & i : getResult()) { auto j = backMap.find(i); assert(j != backMap.end()); found.insert(j->second); @@ -109,6 +100,31 @@ } +std::pair scanForReferences( + const std::string & path, + const StorePathSet & refs) +{ + HashSink hashSink { htSHA256 }; + auto found = scanForReferences(hashSink, path, refs); + auto hash = hashSink.finish(); + return std::pair(found, hash); +} + +StorePathSet scanForReferences( + Sink & toTee, + const Path & path, + const StorePathSet & refs) +{ + PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); + TeeSink sink { refsSink, toTee }; + + /* Look for the hashes in the NAR dump of the path. */ + dumpPath(path, sink); + + return refsSink.getResultPaths(); +} + + RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) : from(from), to(to), nextSink(nextSink) { diff -Nru nix-2.8.0/src/libstore/references.hh nix-2.16.1+dfsg/src/libstore/references.hh --- nix-2.8.0/src/libstore/references.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/references.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "hash.hh" #include "path.hh" @@ -27,6 +28,19 @@ void operator () (std::string_view data) override; }; +class PathRefScanSink : public RefScanSink +{ + std::map backMap; + + PathRefScanSink(StringSet && hashes, std::map && backMap); + +public: + + static PathRefScanSink fromPaths(const StorePathSet & refs); + + StorePathSet getResultPaths(); +}; + struct RewritingSink : Sink { std::string from, to, prev; diff -Nru nix-2.8.0/src/libstore/remote-fs-accessor.cc nix-2.16.1+dfsg/src/libstore/remote-fs-accessor.cc --- nix-2.8.0/src/libstore/remote-fs-accessor.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/remote-fs-accessor.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,6 @@ +#include #include "remote-fs-accessor.hh" #include "nar-accessor.hh" -#include "json.hh" #include #include @@ -38,10 +38,8 @@ if (cacheDir != "") { try { - std::ostringstream str; - JSONPlaceholder jsonRoot(str); - listNar(jsonRoot, narAccessor, "", true); - writeFile(makeCacheFile(hashPart, "ls"), str.str()); + nlohmann::json j = listNar(narAccessor, "", true); + writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreException(); } diff -Nru nix-2.8.0/src/libstore/remote-fs-accessor.hh nix-2.16.1+dfsg/src/libstore/remote-fs-accessor.hh --- nix-2.8.0/src/libstore/remote-fs-accessor.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/remote-fs-accessor.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "fs-accessor.hh" #include "ref.hh" diff -Nru nix-2.8.0/src/libstore/remote-store.cc nix-2.16.1+dfsg/src/libstore/remote-store.cc --- nix-2.8.0/src/libstore/remote-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/remote-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -18,133 +18,6 @@ namespace nix { -namespace worker_proto { - -std::string read(const Store & store, Source & from, Phantom _) -{ - return readString(from); -} - -void write(const Store & store, Sink & out, const std::string & str) -{ - out << str; -} - - -StorePath read(const Store & store, Source & from, Phantom _) -{ - return store.parseStorePath(readString(from)); -} - -void write(const Store & store, Sink & out, const StorePath & storePath) -{ - out << store.printStorePath(storePath); -} - - -ContentAddress read(const Store & store, Source & from, Phantom _) -{ - return parseContentAddress(readString(from)); -} - -void write(const Store & store, Sink & out, const ContentAddress & ca) -{ - out << renderContentAddress(ca); -} - - -DerivedPath read(const Store & store, Source & from, Phantom _) -{ - auto s = readString(from); - return DerivedPath::parse(store, s); -} - -void write(const Store & store, Sink & out, const DerivedPath & req) -{ - out << req.to_string(store); -} - - -Realisation read(const Store & store, Source & from, Phantom _) -{ - std::string rawInput = readString(from); - return Realisation::fromJSON( - nlohmann::json::parse(rawInput), - "remote-protocol" - ); -} - -void write(const Store & store, Sink & out, const Realisation & realisation) -{ - out << realisation.toJSON().dump(); -} - - -DrvOutput read(const Store & store, Source & from, Phantom _) -{ - return DrvOutput::parse(readString(from)); -} - -void write(const Store & store, Sink & out, const DrvOutput & drvOutput) -{ - out << drvOutput.to_string(); -} - - -BuildResult read(const Store & store, Source & from, Phantom _) -{ - auto path = worker_proto::read(store, from, Phantom {}); - BuildResult res { .path = path }; - res.status = (BuildResult::Status) readInt(from); - from - >> res.errorMsg - >> res.timesBuilt - >> res.isNonDeterministic - >> res.startTime - >> res.stopTime; - res.builtOutputs = worker_proto::read(store, from, Phantom {}); - return res; -} - -void write(const Store & store, Sink & to, const BuildResult & res) -{ - worker_proto::write(store, to, res.path); - to - << res.status - << res.errorMsg - << res.timesBuilt - << res.isNonDeterministic - << res.startTime - << res.stopTime; - worker_proto::write(store, to, res.builtOutputs); -} - - -std::optional read(const Store & store, Source & from, Phantom> _) -{ - auto s = readString(from); - return s == "" ? std::optional {} : store.parseStorePath(s); -} - -void write(const Store & store, Sink & out, const std::optional & storePathOpt) -{ - out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); -} - - -std::optional read(const Store & store, Source & from, Phantom> _) -{ - return parseContentAddressOpt(readString(from)); -} - -void write(const Store & store, Sink & out, const std::optional & caOpt) -{ - out << (caOpt ? renderContentAddress(*caOpt) : ""); -} - -} - - /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : RemoteStoreConfig(params) @@ -226,6 +99,13 @@ conn.daemonNixVersion = readString(conn.from); } + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) { + conn.remoteTrustsUs = WorkerProto>::read(*this, conn.from); + } else { + // We don't know the answer; protocol to old. + conn.remoteTrustsUs = std::nullopt; + } + auto ex = conn.processStderr(); if (ex) std::rethrow_exception(ex); } @@ -265,7 +145,8 @@ overrides.erase(settings.buildCores.name); overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); - overrides.erase(settings.experimentalFeatures.name); + overrides.erase(experimentalFeatureSettings.experimentalFeatures.name); + overrides.erase(settings.pluginFiles.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; @@ -346,12 +227,12 @@ return res; } else { conn->to << wopQueryValidPaths; - worker_proto::write(*this, conn->to, paths); + workerProtoWrite(*this, conn->to, paths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { conn->to << (settings.buildersUseSubstitutes ? 1 : 0); } conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } } @@ -361,7 +242,7 @@ auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } @@ -378,9 +259,9 @@ return res; } else { conn->to << wopQuerySubstitutablePaths; - worker_proto::write(*this, conn->to, paths); + workerProtoWrite(*this, conn->to, paths); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } } @@ -402,7 +283,7 @@ auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references = WorkerProto::read(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -415,9 +296,9 @@ StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); - worker_proto::write(*this, conn->to, paths); + workerProtoWrite(*this, conn->to, paths); } else - worker_proto::write(*this, conn->to, pathsMap); + workerProtoWrite(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { @@ -425,7 +306,7 @@ auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references = WorkerProto::read(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -447,7 +328,7 @@ } catch (Error & e) { // Ugly backwards compatibility hack. if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.info()); + throw InvalidPath(std::move(e.info())); throw; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { @@ -468,7 +349,7 @@ auto conn(getConnection()); conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) + for (auto & i : WorkerProto::read(*this, conn->from)) referrers.insert(i); } @@ -478,7 +359,7 @@ auto conn(getConnection()); conn->to << wopQueryValidDerivers << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } @@ -490,7 +371,7 @@ auto conn(getConnection()); conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } @@ -500,7 +381,7 @@ auto conn(getConnection()); conn->to << wopQueryDerivationOutputMap << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom>> {}); + return WorkerProto>>::read(*this, conn->from); } else { // Fallback for old daemon versions. // For floating-CA derivations (and their co-dependencies) this is an @@ -533,6 +414,7 @@ Source & dump, std::string_view name, ContentAddressMethod caMethod, + HashType hashType, const StorePathSet & references, RepairFlag repair) { @@ -544,8 +426,8 @@ conn->to << wopAddToStore << name - << renderContentAddressMethod(caMethod); - worker_proto::write(*this, conn->to, references); + << caMethod.render(hashType); + workerProtoWrite(*this, conn->to, references); conn->to << repair; // The dump source may invoke the store, so we need to make some room. @@ -564,34 +446,35 @@ if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); std::visit(overloaded { - [&](const TextHashMethod & thm) -> void { + [&](const TextIngestionMethod & thm) -> void { + if (hashType != htSHA256) + throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", + name, printHashType(hashType)); std::string s = dump.drain(); conn->to << wopAddTextToStore << name << s; - worker_proto::write(*this, conn->to, references); + workerProtoWrite(*this, conn->to, references); conn.processStderr(); }, - [&](const FixedOutputHashMethod & fohm) -> void { + [&](const FileIngestionMethod & fim) -> void { conn->to << wopAddToStore << name - << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(fohm.hashType); + << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (fim == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(hashType); try { conn->to.written = 0; - conn->to.warn = true; connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); - if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { + if (fim == FileIngestionMethod::Recursive) { dump.drainInto(conn->to); } else { std::string contents = dump.drain(); dumpString(contents, conn->to); } } - conn->to.warn = false; conn.processStderr(); } catch (SysError & e) { /* Daemon closed while we were sending the path. Probably OOM @@ -604,7 +487,7 @@ } } - }, caMethod); + }, caMethod.raw); auto path = parseStorePath(readString(conn->from)); // Release our connection to prevent a deadlock in queryPathInfo(). conn_.reset(); @@ -616,7 +499,7 @@ StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) { - return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; + return addCAToStore(dump, name, method, hashType, references, repair)->path; } @@ -635,7 +518,7 @@ sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.references); + workerProtoWrite(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -645,7 +528,7 @@ conn.processStderr(0, source2.get()); - auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); + auto importedPaths = WorkerProto::read(*this, conn->from); assert(importedPaths.size() <= 1); } @@ -654,7 +537,7 @@ << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + workerProtoWrite(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -674,6 +557,23 @@ void RemoteStore::addMultipleToStore( + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + auto source = sinkToSource([&](Sink & sink) { + sink << pathsToCopy.size(); + for (auto & [pathInfo, pathSource] : pathsToCopy) { + pathInfo.write(sink, *this, 16); + pathSource->drainInto(sink); + } + }); + + addMultipleToStore(*source, repair, checkSigs); +} + +void RemoteStore::addMultipleToStore( Source & source, RepairFlag repair, CheckSigsFlag checkSigs) @@ -699,7 +599,7 @@ RepairFlag repair) { StringSource source(s); - return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; + return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path; } void RemoteStore::registerDrvOutput(const Realisation & info) @@ -710,7 +610,7 @@ conn->to << info.id.to_string(); conn->to << std::string(info.outPath.to_string()); } else { - worker_proto::write(*this, conn->to, info); + workerProtoWrite(*this, conn->to, info); } conn.processStderr(); } @@ -718,36 +618,34 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, Callback> callback) noexcept { - auto conn(getConnection()); + try { + auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { - warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); - try { - callback(nullptr); - } catch (...) { return callback.rethrow(); } - } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { + warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); + return callback(nullptr); + } - conn->to << wopQueryRealisation; - conn->to << id.to_string(); - conn.processStderr(); + conn->to << wopQueryRealisation; + conn->to << id.to_string(); + conn.processStderr(); - auto real = [&]() -> std::shared_ptr { - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { - auto outPaths = worker_proto::read( - *this, conn->from, Phantom> {}); - if (outPaths.empty()) - return nullptr; - return std::make_shared(Realisation { .id = id, .outPath = *outPaths.begin() }); - } else { - auto realisations = worker_proto::read( - *this, conn->from, Phantom> {}); - if (realisations.empty()) - return nullptr; - return std::make_shared(*realisations.begin()); - } - }(); + auto real = [&]() -> std::shared_ptr { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + auto outPaths = WorkerProto>::read( + *this, conn->from); + if (outPaths.empty()) + return nullptr; + return std::make_shared(Realisation { .id = id, .outPath = *outPaths.begin() }); + } else { + auto realisations = WorkerProto>::read( + *this, conn->from); + if (realisations.empty()) + return nullptr; + return std::make_shared(*realisations.begin()); + } + }(); - try { callback(std::shared_ptr(real)); } catch (...) { return callback.rethrow(); } } @@ -755,7 +653,7 @@ static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) { - worker_proto::write(store, conn->to, reqs); + workerProtoWrite(store, conn->to, reqs); } else { Strings ss; for (auto & p : reqs) { @@ -810,7 +708,7 @@ readInt(conn->from); } -std::vector RemoteStore::buildPathsWithResults( +std::vector RemoteStore::buildPathsWithResults( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) @@ -825,7 +723,7 @@ writeDerivedPaths(*this, conn, paths); conn->to << buildMode; conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom> {}); + return WorkerProto>::read(*this, conn->from); } else { // Avoid deadlock. conn_.reset(); @@ -834,53 +732,50 @@ // fails, but meh. buildPaths(paths, buildMode, evalStore); - std::vector results; + std::vector results; for (auto & path : paths) { std::visit( overloaded { [&](const DerivedPath::Opaque & bo) { - results.push_back(BuildResult { - .status = BuildResult::Substituted, - .path = bo, + results.push_back(KeyedBuildResult { + { + .status = BuildResult::Substituted, + }, + /* .path = */ bo, }); }, [&](const DerivedPath::Built & bfd) { - BuildResult res { - .status = BuildResult::Built, - .path = bfd, + KeyedBuildResult res { + { + .status = BuildResult::Built + }, + /* .path = */ bfd, }; OutputPathMap outputs; auto drv = evalStore->readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive - auto drvOutputs = drv.outputsAndOptPaths(*this); - for (auto & output : bfd.outputs) { - if (!outputHashes.count(output)) + const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive + auto built = resolveDerivedPath(*this, bfd, &*evalStore); + for (auto & [output, outputPath] : built) { + auto outputHash = get(outputHashes, output); + if (!outputHash) throw Error( "the derivation '%s' doesn't have an output named '%s'", printStorePath(bfd.drvPath), output); - auto outputId = - DrvOutput{outputHashes.at(output), output}; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + auto outputId = DrvOutput{ *outputHash, output }; + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = queryRealisation(outputId); if (!realisation) - throw Error( - "cannot operate on an output of unbuilt " - "content-addressed derivation '%s'", - outputId.to_string()); - res.builtOutputs.emplace(realisation->id, *realisation); + throw MissingRealisation(outputId); + res.builtOutputs.emplace(output, *realisation); } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - assert(drvOutputs.count(output)); - assert(drvOutputs.at(output).second); res.builtOutputs.emplace( - outputId, + output, Realisation { .id = outputId, - .outPath = *drvOutputs.at(output).second + .outPath = outputPath, }); } } @@ -904,15 +799,18 @@ writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - BuildResult res { .path = DerivedPath::Built { .drvPath = drvPath } }; + BuildResult res; res.status = (BuildResult::Status) readInt(conn->from); conn->from >> res.errorMsg; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { - auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); - res.builtOutputs = builtOutputs; + auto builtOutputs = WorkerProto::read(*this, conn->from); + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); } return res; } @@ -967,7 +865,7 @@ conn->to << wopCollectGarbage << options.action; - worker_proto::write(*this, conn->to, options.pathsToDelete); + workerProtoWrite(*this, conn->to, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -1026,9 +924,9 @@ conn->to << wopQueryMissing; writeDerivedPaths(*this, conn, targets); conn.processStderr(); - willBuild = worker_proto::read(*this, conn->from, Phantom {}); - willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); - unknown = worker_proto::read(*this, conn->from, Phantom {}); + willBuild = WorkerProto::read(*this, conn->from); + willSubstitute = WorkerProto::read(*this, conn->from); + unknown = WorkerProto::read(*this, conn->from); conn->from >> downloadSize >> narSize; return; } @@ -1070,6 +968,11 @@ return conn->daemonVersion; } +std::optional RemoteStore::isTrustedClient() +{ + auto conn(getConnection()); + return conn->remoteTrustsUs; +} void RemoteStore::flushBadConnections() { diff -Nru nix-2.8.0/src/libstore/remote-store.hh nix-2.16.1+dfsg/src/libstore/remote-store.hh --- nix-2.8.0/src/libstore/remote-store.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/remote-store.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -22,15 +23,19 @@ { using StoreConfig::StoreConfig; - const Setting maxConnections{(StoreConfig*) this, 1, - "max-connections", "maximum number of concurrent connections to the Nix daemon"}; + const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + "Maximum number of concurrent connections to the Nix daemon."}; - const Setting maxConnectionAge{(StoreConfig*) this, std::numeric_limits::max(), - "max-connection-age", "number of seconds to reuse a connection"}; + const Setting maxConnectionAge{(StoreConfig*) this, + std::numeric_limits::max(), + "max-connection-age", + "Maximum age of a connection before it is closed."}; }; -/* FIXME: RemoteStore is a misnomer - should be something like - DaemonStore. */ +/** + * \todo RemoteStore is a misnomer - should be something like + * DaemonStore. + */ class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore, @@ -38,8 +43,6 @@ { public: - virtual bool sameMachine() = 0; - RemoteStore(const Params & params); /* Implementations of abstract store API methods. */ @@ -68,15 +71,20 @@ void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - /* Add a content-addressable store path. `dump` will be drained. */ + /** + * Add a content-addressable store path. `dump` will be drained. + */ ref addCAToStore( Source & dump, std::string_view name, ContentAddressMethod caMethod, + HashType hashType, const StorePathSet & references, RepairFlag repair); - /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ + /** + * Add a content-addressable store path. Does not support references. `dump` will be drained. + */ StorePath addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; @@ -88,6 +96,12 @@ RepairFlag repair, CheckSigsFlag checkSigs) override; + void addMultipleToStore( + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) override; + StorePath addTextToStore( std::string_view name, std::string_view s, @@ -101,7 +115,7 @@ void buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; - std::vector buildPathsWithResults( + std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; @@ -123,6 +137,17 @@ bool verifyStore(bool checkContents, RepairFlag repair) override; + /** + * The default instance would schedule the work on the client side, but + * for consistency with `buildPaths` and `buildDerivation` it should happen + * on the remote side. + * + * We make this fail for now so we can add implement this properly later + * without it being a breaking change. + */ + void repairPath(const StorePath & path) override + { unsupported("repairPath"); } + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; void queryMissing(const std::vector & targets, @@ -137,6 +162,8 @@ unsigned int getProtocol() override; + std::optional isTrustedClient() override; + void flushBadConnections(); struct Connection @@ -144,6 +171,7 @@ FdSink to; FdSource from; unsigned int daemonVersion; + std::optional remoteTrustsUs; std::optional daemonNixVersion; std::chrono::time_point startTime; diff -Nru nix-2.8.0/src/libstore/repair-flag.hh nix-2.16.1+dfsg/src/libstore/repair-flag.hh --- nix-2.8.0/src/libstore/repair-flag.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/repair-flag.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff -Nru nix-2.8.0/src/libstore/s3-binary-cache-store.cc nix-2.16.1+dfsg/src/libstore/s3-binary-cache-store.cc --- nix-2.8.0/src/libstore/s3-binary-cache-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/s3-binary-cache-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -40,12 +40,12 @@ /* Helper: given an Outcome, return R in case of success, or throw an exception in case of an error. */ template -R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome && outcome) +R && checkAws(std::string_view s, Aws::Utils::Outcome && outcome) { if (!outcome.IsSuccess()) throw S3Error( outcome.GetError().GetErrorType(), - fs.s + ": " + outcome.GetError().GetMessage()); + s + ": " + outcome.GetError().GetMessage()); return outcome.GetResultWithOwnership(); } @@ -192,19 +192,72 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const Setting profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."}; - const Setting region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; - const Setting scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."}; - const Setting endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; - const Setting narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"}; - const Setting lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"}; - const Setting logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"}; + + const Setting profile{(StoreConfig*) this, "", "profile", + R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )"}; + + const Setting region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", + R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )"}; + + const Setting scheme{(StoreConfig*) this, "", "scheme", + R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )"}; + + const Setting endpoint{(StoreConfig*) this, "", "endpoint", + R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )"}; + + const Setting narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", + "Compression method for `.narinfo` files."}; + + const Setting lsCompression{(StoreConfig*) this, "", "ls-compression", + "Compression method for `.ls` files."}; + + const Setting logCompression{(StoreConfig*) this, "", "log-compression", + R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )"}; + const Setting multipartUpload{ - (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"}; + (StoreConfig*) this, false, "multipart-upload", + "Whether to use multi-part uploads."}; + const Setting bufferSize{ - (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; + (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", + "Size (in bytes) of each part in multi-part uploads."}; const std::string name() override { return "S3 Binary Cache Store"; } + + std::string doc() override + { + return + #include "s3-binary-cache-store.md" + ; + } }; struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore @@ -238,7 +291,7 @@ void init() override { - if (auto cacheInfo = diskCache->cacheExists(getUri())) { + if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) { wantMassQuery.setDefault(cacheInfo->wantMassQuery); priority.setDefault(cacheInfo->priority); } else { @@ -430,9 +483,9 @@ std::string marker; do { - debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker); + debug("listing bucket 's3://%s' from key '%s'...", bucketName, marker); - auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName, + auto res = checkAws(fmt("AWS error listing bucket '%s'", bucketName), s3Helper.client->ListObjects( Aws::S3::Model::ListObjectsRequest() .WithBucket(bucketName) @@ -441,8 +494,8 @@ auto & contents = res.GetContents(); - debug(format("got %d keys, next marker '%s'") - % contents.size() % res.GetNextMarker()); + debug("got %d keys, next marker '%s'", + contents.size(), res.GetNextMarker()); for (auto object : contents) { auto & key = object.GetKey(); @@ -456,6 +509,16 @@ return paths; } + /** + * For now, we conservatively say we don't know. + * + * \todo try to expose our S3 authentication status. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } + static std::set uriSchemes() { return {"s3"}; } }; diff -Nru nix-2.8.0/src/libstore/s3-binary-cache-store.hh nix-2.16.1+dfsg/src/libstore/s3-binary-cache-store.hh --- nix-2.8.0/src/libstore/s3-binary-cache-store.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/s3-binary-cache-store.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "binary-cache-store.hh" diff -Nru nix-2.8.0/src/libstore/s3-binary-cache-store.md nix-2.16.1+dfsg/src/libstore/s3-binary-cache-store.md --- nix-2.8.0/src/libstore/s3-binary-cache-store.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/s3-binary-cache-store.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `s3://`*bucket-name* + +This store allows reading and writing a binary cache stored in an AWS +S3 bucket. + +)" diff -Nru nix-2.8.0/src/libstore/s3.hh nix-2.16.1+dfsg/src/libstore/s3.hh --- nix-2.8.0/src/libstore/s3.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/s3.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,10 +1,12 @@ #pragma once +///@file #if ENABLE_S3 #include "ref.hh" #include +#include namespace Aws { namespace Client { class ClientConfiguration; } } namespace Aws { namespace S3 { class S3Client; } } diff -Nru nix-2.8.0/src/libstore/sandbox-defaults.sb nix-2.16.1+dfsg/src/libstore/sandbox-defaults.sb --- nix-2.8.0/src/libstore/sandbox-defaults.sb 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/sandbox-defaults.sb 1970-01-01 00:00:00.000000000 +0000 @@ -1,104 +0,0 @@ -(define TMPDIR (param "_GLOBAL_TMP_DIR")) - -(deny default) - -; Disallow creating setuid/setgid binaries, since that -; would allow breaking build user isolation. -(deny file-write-setugid) - -; Allow forking. -(allow process-fork) - -; Allow reading system information like #CPUs, etc. -(allow sysctl-read) - -; Allow POSIX semaphores and shared memory. -(allow ipc-posix*) - -; Allow socket creation. -(allow system-socket) - -; Allow sending signals within the sandbox. -(allow signal (target same-sandbox)) - -; Allow getpwuid. -(allow mach-lookup (global-name "com.apple.system.opendirectoryd.libinfo")) - -; Access to /tmp. -; The network-outbound/network-inbound ones are for unix domain sockets, which -; we allow access to in TMPDIR (but if we allow them more broadly, you could in -; theory escape the sandbox) -(allow file* process-exec network-outbound network-inbound - (literal "/tmp") (subpath TMPDIR)) - -; Some packages like to read the system version. -(allow file-read* - (literal "/System/Library/CoreServices/SystemVersion.plist") - (literal "/System/Library/CoreServices/SystemVersionCompat.plist")) - -; Without this line clang cannot write to /dev/null, breaking some configure tests. -(allow file-read-metadata (literal "/dev")) - -; Many packages like to do local networking in their test suites, but let's only -; allow it if the package explicitly asks for it. -(if (param "_ALLOW_LOCAL_NETWORKING") - (begin - (allow network* (local ip) (local tcp) (local udp)) - - ; Allow access to /etc/resolv.conf (which is a symlink to - ; /private/var/run/resolv.conf). - ; TODO: deduplicate with sandbox-network.sb - (allow file-read-metadata - (literal "/var") - (literal "/etc") - (literal "/etc/resolv.conf") - (literal "/private/etc/resolv.conf")) - - (allow file-read* - (literal "/private/var/run/resolv.conf")) - - ; Allow DNS lookups. This is even needed for localhost, which lots of tests rely on - (allow file-read-metadata (literal "/etc/hosts")) - (allow file-read* (literal "/private/etc/hosts")) - (allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))))) - -; Standard devices. -(allow file* - (literal "/dev/null") - (literal "/dev/random") - (literal "/dev/stdin") - (literal "/dev/stdout") - (literal "/dev/tty") - (literal "/dev/urandom") - (literal "/dev/zero") - (subpath "/dev/fd")) - -; Allow pseudo-terminals. -(allow file* - (literal "/dev/ptmx") - (regex #"^/dev/pty[a-z]+") - (regex #"^/dev/ttys[0-9]+")) - -; Does nothing, but reduces build noise. -(allow file* (literal "/dev/dtracehelper")) - -; Allow access to zoneinfo since libSystem needs it. -(allow file-read* (subpath "/usr/share/zoneinfo")) - -(allow file-read* (subpath "/usr/share/locale")) - -; This is mostly to get more specific log messages when builds try to -; access something in /etc or /var. -(allow file-read-metadata - (literal "/etc") - (literal "/var") - (literal "/private/var/tmp")) - -; This is used by /bin/sh on macOS 10.15 and later. -(allow file* - (literal "/private/var/select/sh")) - -; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin. -(allow file-read* - (subpath "/Library/Apple/usr/libexec/oah") - (subpath "/System/Library/Apple/usr/libexec/oah")) diff -Nru nix-2.8.0/src/libstore/sandbox-minimal.sb nix-2.16.1+dfsg/src/libstore/sandbox-minimal.sb --- nix-2.8.0/src/libstore/sandbox-minimal.sb 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/sandbox-minimal.sb 1970-01-01 00:00:00.000000000 +0000 @@ -1,5 +0,0 @@ -(allow default) - -; Disallow creating setuid/setgid binaries, since that -; would allow breaking build user isolation. -(deny file-write-setugid) diff -Nru nix-2.8.0/src/libstore/sandbox-network.sb nix-2.16.1+dfsg/src/libstore/sandbox-network.sb --- nix-2.8.0/src/libstore/sandbox-network.sb 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/sandbox-network.sb 1970-01-01 00:00:00.000000000 +0000 @@ -1,16 +0,0 @@ -; Allow local and remote network traffic. -(allow network* (local ip) (remote ip)) - -; Allow access to /etc/resolv.conf (which is a symlink to -; /private/var/run/resolv.conf). -(allow file-read-metadata - (literal "/var") - (literal "/etc") - (literal "/etc/resolv.conf") - (literal "/private/etc/resolv.conf")) - -(allow file-read* - (literal "/private/var/run/resolv.conf")) - -; Allow DNS lookups. -(allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))) diff -Nru nix-2.8.0/src/libstore/schema.sql nix-2.16.1+dfsg/src/libstore/schema.sql --- nix-2.8.0/src/libstore/schema.sql 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/schema.sql 2023-06-06 10:55:23.000000000 +0000 @@ -1,7 +1,7 @@ create table if not exists ValidPaths ( id integer primary key autoincrement not null, path text unique not null, - hash text not null, + hash text not null, -- base16 representation registrationTime integer not null, deriver text, narSize integer, diff -Nru nix-2.8.0/src/libstore/serve-protocol.hh nix-2.16.1+dfsg/src/libstore/serve-protocol.hh --- nix-2.8.0/src/libstore/serve-protocol.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/serve-protocol.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff -Nru nix-2.8.0/src/libstore/sqlite.cc nix-2.16.1+dfsg/src/libstore/sqlite.cc --- nix-2.8.0/src/libstore/sqlite.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/sqlite.cc 2023-06-06 10:55:23.000000000 +0000 @@ -8,36 +8,69 @@ namespace nix { -[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs) +SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf) + : Error(""), path(path), errMsg(errMsg), errNo(errNo), extendedErrNo(extendedErrNo), offset(offset) +{ + auto offsetStr = (offset == -1) ? "" : "at offset " + std::to_string(offset) + ": "; + err.msg = hintfmt("%s: %s%s, %s (in '%s')", + normaltxt(hf.str()), + offsetStr, + sqlite3_errstr(extendedErrNo), + errMsg, + path ? path : "(in-memory)"); +} + +[[noreturn]] void SQLiteError::throw_(sqlite3 * db, hintformat && hf) { int err = sqlite3_errcode(db); int exterr = sqlite3_extended_errcode(db); + int offset = sqlite3_error_offset(db); auto path = sqlite3_db_filename(db, nullptr); - if (!path) path = "(in-memory)"; + auto errMsg = sqlite3_errmsg(db); if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - throw SQLiteBusy( + auto exp = SQLiteBusy(path, errMsg, err, exterr, offset, std::move(hf)); + exp.err.msg = hintfmt( err == SQLITE_PROTOCOL - ? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path) - : fmt("SQLite database '%s' is busy", path)); - } - else - throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); + ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" + : "SQLite database '%s' is busy", + path ? path : "(in-memory)"); + throw exp; + } else + throw SQLiteError(path, errMsg, err, exterr, offset, std::move(hf)); } +static void traceSQL(void * x, const char * sql) +{ + // wacky delimiters: + // so that we're quite unambiguous without escaping anything + // notice instead of trace: + // so that this can be enabled without getting the firehose in our face. + notice("SQL<[%1%]>", sql); +}; + SQLite::SQLite(const Path & path, bool create) { // useSQLiteWAL also indicates what virtual file system we need. Using // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem // for Linux (WSL) where useSQLiteWAL should be false by default. const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; - if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), vfs) != SQLITE_OK) - throw Error("cannot open SQLite database '%s'", path); + int flags = SQLITE_OPEN_READWRITE; + if (create) flags |= SQLITE_OPEN_CREATE; + int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs); + if (ret != SQLITE_OK) { + const char * err = sqlite3_errstr(ret); + throw Error("cannot open SQLite database '%s': %s", path, err); + } if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(db, "setting timeout"); + SQLiteError::throw_(db, "setting timeout"); + + if (getEnv("NIX_DEBUG_SQLITE_TRACES") == "1") { + // To debug sqlite statements; trace all of them + sqlite3_trace(db, &traceSQL, nullptr); + } exec("pragma foreign_keys = 1"); } @@ -46,7 +79,7 @@ { try { if (db && sqlite3_close(db) != SQLITE_OK) - throwSQLiteError(db, "closing database"); + SQLiteError::throw_(db, "closing database"); } catch (...) { ignoreException(); } @@ -62,7 +95,7 @@ { retrySQLite([&]() { if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt); + SQLiteError::throw_(db, "executing SQLite statement '%s'", stmt); }); } @@ -76,7 +109,7 @@ checkInterrupt(); assert(!stmt); if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, fmt("creating statement '%s'", sql)); + SQLiteError::throw_(db, "creating statement '%s'", sql); this->db = db; this->sql = sql; } @@ -85,7 +118,7 @@ { try { if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, fmt("finalizing statement '%s'", sql)); + SQLiteError::throw_(db, "finalizing statement '%s'", sql); } catch (...) { ignoreException(); } @@ -109,7 +142,7 @@ { if (notNull) { if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); + SQLiteError::throw_(stmt.db, "binding argument"); } else bind(); return *this; @@ -119,7 +152,7 @@ { if (notNull) { if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); + SQLiteError::throw_(stmt.db, "binding argument"); } else bind(); return *this; @@ -129,7 +162,7 @@ { if (notNull) { if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); + SQLiteError::throw_(stmt.db, "binding argument"); } else bind(); return *this; @@ -138,7 +171,7 @@ SQLiteStmt::Use & SQLiteStmt::Use::bind() { if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); + SQLiteError::throw_(stmt.db, "binding argument"); return *this; } @@ -152,14 +185,14 @@ int r = step(); assert(r != SQLITE_ROW); if (r != SQLITE_DONE) - throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", sqlite3_expanded_sql(stmt.stmt))); + SQLiteError::throw_(stmt.db, fmt("executing SQLite statement '%s'", sqlite3_expanded_sql(stmt.stmt))); } bool SQLiteStmt::Use::next() { int r = step(); if (r != SQLITE_DONE && r != SQLITE_ROW) - throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", sqlite3_expanded_sql(stmt.stmt))); + SQLiteError::throw_(stmt.db, fmt("executing SQLite query '%s'", sqlite3_expanded_sql(stmt.stmt))); return r == SQLITE_ROW; } @@ -185,14 +218,14 @@ { this->db = db; if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "starting transaction"); + SQLiteError::throw_(db, "starting transaction"); active = true; } void SQLiteTxn::commit() { if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "committing transaction"); + SQLiteError::throw_(db, "committing transaction"); active = false; } @@ -200,20 +233,17 @@ { try { if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "aborting transaction"); + SQLiteError::throw_(db, "aborting transaction"); } catch (...) { ignoreException(); } } -void handleSQLiteBusy(const SQLiteBusy & e) +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning) { - static std::atomic lastWarned{0}; - time_t now = time(0); - - if (now > lastWarned + 10) { - lastWarned = now; + if (now > nextWarning) { + nextWarning = now + 10; logWarning({ .msg = hintfmt(e.what()) }); diff -Nru nix-2.8.0/src/libstore/sqlite.hh nix-2.16.1+dfsg/src/libstore/sqlite.hh --- nix-2.8.0/src/libstore/sqlite.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/sqlite.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -10,7 +11,9 @@ namespace nix { -/* RAII wrapper to close a SQLite database automatically. */ +/** + * RAII wrapper to close a SQLite database automatically. + */ struct SQLite { sqlite3 * db = 0; @@ -22,7 +25,9 @@ ~SQLite(); operator sqlite3 * () { return db; } - /* Disable synchronous mode, set truncate journal mode. */ + /** + * Disable synchronous mode, set truncate journal mode. + */ void isCache(); void exec(const std::string & stmt); @@ -30,7 +35,9 @@ uint64_t getLastInsertedRowId(); }; -/* RAII wrapper to create and destroy SQLite prepared statements. */ +/** + * RAII wrapper to create and destroy SQLite prepared statements. + */ struct SQLiteStmt { sqlite3 * db = 0; @@ -42,7 +49,9 @@ ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } - /* Helper for binding / executing statements. */ + /** + * Helper for binding / executing statements. + */ class Use { friend struct SQLiteStmt; @@ -55,7 +64,9 @@ ~Use(); - /* Bind the next parameter. */ + /** + * Bind the next parameter. + */ Use & operator () (std::string_view value, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); @@ -63,11 +74,15 @@ int step(); - /* Execute a statement that does not return rows. */ + /** + * Execute a statement that does not return rows. + */ void exec(); - /* For statements that return 0 or more rows. Returns true iff - a row is available. */ + /** + * For statements that return 0 or more rows. Returns true iff + * a row is available. + */ bool next(); std::string getStr(int col); @@ -81,8 +96,10 @@ } }; -/* RAII helper that ensures transactions are aborted unless explicitly - committed. */ +/** + * RAII helper that ensures transactions are aborted unless explicitly + * committed. + */ struct SQLiteTxn { bool active = false; @@ -96,23 +113,48 @@ }; -MakeError(SQLiteError, Error); -MakeError(SQLiteBusy, SQLiteError); +struct SQLiteError : Error +{ + std::string path; + std::string errMsg; + int errNo, extendedErrNo, offset; + + template + [[noreturn]] static void throw_(sqlite3 * db, const std::string & fs, const Args & ... args) { + throw_(db, hintfmt(fs, args...)); + } -[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs); + SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf); -void handleSQLiteBusy(const SQLiteBusy & e); +protected: -/* Convenience function for retrying a SQLite transaction when the - database is busy. */ + template + SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, const std::string & fs, const Args & ... args) + : SQLiteError(path, errNo, extendedErrNo, offset, hintfmt(fs, args...)) + { } + + [[noreturn]] static void throw_(sqlite3 * db, hintformat && hf); + +}; + +MakeError(SQLiteBusy, SQLiteError); + +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning); + +/** + * Convenience function for retrying a SQLite transaction when the + * database is busy. + */ template T retrySQLite(F && fun) { + time_t nextWarning = time(0) + 1; + while (true) { try { return fun(); } catch (SQLiteBusy & e) { - handleSQLiteBusy(e); + handleSQLiteBusy(e, nextWarning); } } } diff -Nru nix-2.8.0/src/libstore/ssh.cc nix-2.16.1+dfsg/src/libstore/ssh.cc --- nix-2.8.0/src/libstore/ssh.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/ssh.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #include "ssh.hh" +#include "finally.hh" namespace nix { @@ -35,6 +36,14 @@ } if (compress) args.push_back("-C"); + + args.push_back("-oPermitLocalCommand=yes"); + args.push_back("-oLocalCommand=echo started"); +} + +bool SSHMaster::isMasterRunning() { + auto res = runProgram(RunOptions {.program = "ssh", .args = {"-O", "check", host}, .mergeStderrToStdout = true}); + return res.first == 0; } std::unique_ptr SSHMaster::startCommand(const std::string & command) @@ -49,6 +58,11 @@ ProcessOptions options; options.dieWithParent = false; + if (!fakeSSH && !useMaster) { + logger->pause(); + } + Finally cleanup = [&]() { logger->resume(); }; + conn->sshPid = startProcess([&]() { restoreProcessContext(); @@ -67,7 +81,7 @@ if (fakeSSH) { args = { "bash", "-c" }; } else { - args = { "ssh", host.c_str(), "-x", "-a" }; + args = { "ssh", host.c_str(), "-x" }; addCommonSSHOpts(args); if (socketPath != "") args.insert(args.end(), {"-S", socketPath}); @@ -86,6 +100,18 @@ in.readSide = -1; out.writeSide = -1; + // Wait for the SSH connection to be established, + // So that we don't overwrite the password prompt with our progress bar. + if (!fakeSSH && !useMaster && !isMasterRunning()) { + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } + + if (reply != "started") + throw Error("failed to start SSH connection to '%s'", host); + } + conn->out = std::move(out.readSide); conn->in = std::move(in.writeSide); @@ -109,6 +135,11 @@ ProcessOptions options; options.dieWithParent = false; + logger->pause(); + Finally cleanup = [&]() { logger->resume(); }; + + bool wasMasterRunning = isMasterRunning(); + state->sshMaster = startProcess([&]() { restoreProcessContext(); @@ -117,11 +148,7 @@ if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("duping over stdout"); - Strings args = - { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath - , "-o", "LocalCommand=echo started" - , "-o", "PermitLocalCommand=yes" - }; + Strings args = { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath }; if (verbosity >= lvlChatty) args.push_back("-v"); addCommonSSHOpts(args); @@ -132,13 +159,15 @@ out.writeSide = -1; - std::string reply; - try { - reply = readLine(out.readSide.get()); - } catch (EndOfFile & e) { } + if (!wasMasterRunning) { + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } - if (reply != "started") - throw Error("failed to start SSH master connection to '%s'", host); + if (reply != "started") + throw Error("failed to start SSH master connection to '%s'", host); + } return state->socketPath; } diff -Nru nix-2.8.0/src/libstore/ssh.hh nix-2.16.1+dfsg/src/libstore/ssh.hh --- nix-2.8.0/src/libstore/ssh.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/ssh.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "sync.hh" @@ -27,6 +28,7 @@ Sync state_; void addCommonSSHOpts(Strings & args); + bool isMasterRunning(); public: diff -Nru nix-2.8.0/src/libstore/ssh-store.cc nix-2.16.1+dfsg/src/libstore/ssh-store.cc --- nix-2.8.0/src/libstore/ssh-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/ssh-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,3 +1,4 @@ +#include "ssh-store-config.hh" #include "store-api.hh" #include "remote-store.hh" #include "remote-fs-accessor.hh" @@ -8,17 +9,22 @@ namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig +struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig { using RemoteStoreConfig::RemoteStoreConfig; + using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; - const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; - const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", + "Path to the `nix-daemon` executable on the remote machine."}; - const std::string name() override { return "SSH Store"; } + const std::string name() override { return "Experimental SSH Store"; } + + std::string doc() override + { + return + #include "ssh-store.md" + ; + } }; class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore @@ -28,6 +34,7 @@ SSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) , SSHStoreConfig(params) , Store(params) , RemoteStore(params) @@ -49,12 +56,9 @@ return *uriSchemes().begin() + "://" + host; } - bool sameMachine() override - { return false; } - // FIXME extend daemon protocol, move implementation to RemoteStore - std::optional getBuildLog(const StorePath & path) override - { unsupported("getBuildLog"); } + std::optional getBuildLogExact(const StorePath & path) override + { unsupported("getBuildLogExact"); } private: diff -Nru nix-2.8.0/src/libstore/ssh-store-config.hh nix-2.16.1+dfsg/src/libstore/ssh-store-config.hh --- nix-2.8.0/src/libstore/ssh-store-config.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/ssh-store-config.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,29 @@ +#pragma once +///@file + +#include "store-api.hh" + +namespace nix { + +struct CommonSSHStoreConfig : virtual StoreConfig +{ + using StoreConfig::StoreConfig; + + const Setting sshKey{(StoreConfig*) this, "", "ssh-key", + "Path to the SSH private key used to authenticate to the remote machine."}; + + const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", + "The public host key of the remote machine."}; + + const Setting compress{(StoreConfig*) this, false, "compress", + "Whether to enable SSH compression."}; + + const Setting remoteStore{(StoreConfig*) this, "", "remote-store", + R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )"}; +}; + +} diff -Nru nix-2.8.0/src/libstore/ssh-store.md nix-2.16.1+dfsg/src/libstore/ssh-store.md --- nix-2.8.0/src/libstore/ssh-store.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/ssh-store.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `ssh-ng://[username@]hostname` + +Experimental store type that allows full access to a Nix store on a +remote machine. + +)" diff -Nru nix-2.8.0/src/libstore/store-api.cc nix-2.16.1+dfsg/src/libstore/store-api.cc --- nix-2.8.0/src/libstore/store-api.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/store-api.cc 2023-06-06 10:55:23.000000000 +0000 @@ -6,32 +6,35 @@ #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" -#include "json.hh" #include "url.hh" +#include "references.hh" #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +#include #include +using json = nlohmann::json; + namespace nix { -bool Store::isInStore(const Path & path) const +bool Store::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair Store::toStorePath(const Path & path) const +std::pair Store::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); - Path::size_type slash = path.find('/', storeDir.size() + 1); + auto slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) return {parseStorePath(path), ""}; else - return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)}; + return {parseStorePath(path.substr(0, slash)), (Path) path.substr(slash)}; } @@ -96,10 +99,12 @@ silly, but it's done that way for compatibility). is the name of the output (usually, "out"). -

= base-16 representation of a SHA-256 hash of: +

= base-16 representation of a SHA-256 hash of + + = if = "text:...": the string written to the resulting store path - if = "source": + if = "source:...": the serialisation of the path from which this store path is copied, as returned by hashPath() if = "output:": @@ -160,63 +165,63 @@ } +/* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in, say, (per the grammar above) + since that would be ambiguous. */ static std::string makeType( const Store & store, std::string && type, - const StorePathSet & references, - bool hasSelfReference = false) + const StoreReferences & references) { - for (auto & i : references) { + for (auto & i : references.others) { type += ":"; type += store.printStorePath(i); } - if (hasSelfReference) type += ":self"; + if (references.self) type += ":self"; return std::move(type); } -StorePath Store::makeFixedOutputPath( - FileIngestionMethod method, - const Hash & hash, - std::string_view name, - const StorePathSet & references, - bool hasSelfReference) const +StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name); } else { - assert(references.empty()); + assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.hash.method) + + info.hash.hash.to_string(Base16, true) + ":"), name); } } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references, bool hasSelfReference) const + +StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +{ + assert(info.hash.hash.type == htSHA256); + return makeStorePath( + makeType(*this, "text", StoreReferences { + .others = info.references, + .self = false, + }), + info.hash.hash, + name); +} + + +StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { - [&](const TextHash & th) { - return makeTextPath(name, th.hash, references); + [&](const TextInfo & ti) { + return makeTextPath(name, ti); }, - [&](const FixedOutputHash & fsh) { - return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + [&](const FixedOutputInfo & foi) { + return makeFixedOutputPath(name, foi); } - }, ca); -} - -StorePath Store::makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - return makeStorePath(makeType(*this, "text", references), hash, name); + }, ca.raw); } @@ -226,7 +231,14 @@ Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(method, h, name), h); + FixedOutputInfo caInfo { + .hash = { + .method = method, + .hash = h, + }, + .references = {}, + }; + return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -235,7 +247,10 @@ std::string_view s, const StorePathSet & references) const { - return makeTextPath(name, hashString(htSHA256, s), references); + return makeTextPath(name, TextInfo { + { .hash = hashString(htSHA256, s) }, + references, + }); } @@ -258,6 +273,84 @@ return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); } +void Store::addMultipleToStore( + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + std::atomic nrDone{0}; + std::atomic nrFailed{0}; + std::atomic bytesExpected{0}; + std::atomic nrRunning{0}; + + using PathWithInfo = std::pair>; + + std::map infosMap; + StorePathSet storePathsToAdd; + for (auto & thingToAdd : pathsToCopy) { + infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd); + storePathsToAdd.insert(thingToAdd.first.path); + } + + auto showProgress = [&]() { + act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed); + }; + + ThreadPool pool; + + processGraph(pool, + storePathsToAdd, + + [&](const StorePath & path) { + + auto & [info, _] = *infosMap.at(path); + + if (isValidPath(info.path)) { + nrDone++; + showProgress(); + return StorePathSet(); + } + + bytesExpected += info.narSize; + act.setExpected(actCopyPath, bytesExpected); + + return info.references; + }, + + [&](const StorePath & path) { + checkInterrupt(); + + auto & [info_, source_] = *infosMap.at(path); + auto info = info_; + info.ultimate = false; + + /* Make sure that the Source object is destroyed when + we're done. In particular, a SinkToSource object must + be destroyed to ensure that the destructors on its + stack frame are run; this includes + LegacySSHStore::narFromPath()'s connection lock. */ + auto source = std::move(source_); + + if (!isValidPath(info.path)) { + MaintainCount mc(nrRunning); + showProgress(); + try { + addToStore(info, *source, repair, checkSigs); + } catch (Error & e) { + nrFailed++; + if (!settings.keepGoing) + throw e; + printMsg(lvlError, "could not copy %s: %s", printStorePath(path), e.what()); + showProgress(); + return; + } + } + + nrDone++; + showProgress(); + }); +} void Store::addMultipleToStore( Source & source, @@ -345,11 +438,18 @@ throw Error("hash mismatch for '%s'", srcPath); ValidPathInfo info { - makeFixedOutputPath(method, hash, name), + *this, + name, + FixedOutputInfo { + .hash = { + .method = method, + .hash = hash, + }, + .references = {}, + }, narHash, }; info.narSize = narSize; - info.ca = FixedOutputHash { .method = method, .hash = hash }; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { @@ -365,10 +465,10 @@ { auto res = settings.systemFeatures.get(); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) res.insert("ca-derivations"); - if (settings.isExperimentalFeatureEnabled(Xp::RecursiveNix)) + if (experimentalFeatureSettings.isEnabled(Xp::RecursiveNix)) res.insert("recursive-nix"); return res; @@ -378,6 +478,7 @@ : StoreConfig(params) , state({(size_t) pathInfoCacheSize}) { + assertLibStoreInitialized(); } @@ -426,6 +527,57 @@ return outputPaths; } + +void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) +{ + if (!settings.useSubstitutes) return; + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (infos.count(path.first)) + // Choose first succeeding substituter. + continue; + + auto subPath(path.first); + + // Recompute store path so that we can use a different store root. + if (path.second) { + subPath = makeFixedOutputPathFromCA( + path.first.name(), + ContentAddressWithReferences::withoutRefs(*path.second)); + if (sub->storeDir == storeDir) + assert(subPath == path.first); + if (subPath != path.first) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); + } else if (sub->storeDir != storeDir) continue; + + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); + try { + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + continue; + + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + infos.insert_or_assign(path.first, SubstitutablePathInfo{ + .deriver = info->deriver, + .references = info->references, + .downloadSize = narInfo ? narInfo->fileSize : 0, + .narSize = info->narSize, + }); + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + logError(e.info()); + else + throw; + } + } + } +} + + bool Store::isValidPath(const StorePath & storePath) { { @@ -661,13 +813,13 @@ std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -679,7 +831,7 @@ }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); @@ -709,13 +861,13 @@ if (showHash) { s += info->narHash.to_string(Base16, false) + "\n"; - s += (format("%1%\n") % info->narSize).str(); + s += fmt("%1%\n", info->narSize); } auto deriver = showDerivers && info->deriver ? printStorePath(*info->deriver) : ""; s += deriver + "\n"; - s += (format("%1%\n") % info->references.size()).str(); + s += fmt("%1%\n", info->references.size()); for (auto & j : info->references) s += printStorePath(j) + "\n"; @@ -760,56 +912,54 @@ return paths; } - -void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, +json Store::pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, Base hashBase, AllowInvalidFlag allowInvalid) { - auto jsonList = jsonOut.list(); + json::array_t jsonList = json::array(); for (auto & storePath : storePaths) { - auto jsonPath = jsonList.object(); + auto& jsonPath = jsonList.emplace_back(json::object()); try { auto info = queryPathInfo(storePath); - jsonPath.attr("path", printStorePath(info->path)); - jsonPath - .attr("narHash", info->narHash.to_string(hashBase, true)) - .attr("narSize", info->narSize); + jsonPath["path"] = printStorePath(info->path); + jsonPath["valid"] = true; + jsonPath["narHash"] = info->narHash.to_string(hashBase, true); + jsonPath["narSize"] = info->narSize; { - auto jsonRefs = jsonPath.list("references"); + auto& jsonRefs = (jsonPath["references"] = json::array()); for (auto & ref : info->references) - jsonRefs.elem(printStorePath(ref)); + jsonRefs.emplace_back(printStorePath(ref)); } if (info->ca) - jsonPath.attr("ca", renderContentAddress(info->ca)); + jsonPath["ca"] = renderContentAddress(info->ca); std::pair closureSizes; if (showClosureSize) { closureSizes = getClosureSize(info->path); - jsonPath.attr("closureSize", closureSizes.first); + jsonPath["closureSize"] = closureSizes.first; } if (includeImpureInfo) { if (info->deriver) - jsonPath.attr("deriver", printStorePath(*info->deriver)); + jsonPath["deriver"] = printStorePath(*info->deriver); if (info->registrationTime) - jsonPath.attr("registrationTime", info->registrationTime); + jsonPath["registrationTime"] = info->registrationTime; if (info->ultimate) - jsonPath.attr("ultimate", info->ultimate); + jsonPath["ultimate"] = info->ultimate; if (!info->sigs.empty()) { - auto jsonSigs = jsonPath.list("signatures"); for (auto & sig : info->sigs) - jsonSigs.elem(sig); + jsonPath["signatures"].push_back(sig); } auto narInfo = std::dynamic_pointer_cast( @@ -817,21 +967,22 @@ if (narInfo) { if (!narInfo->url.empty()) - jsonPath.attr("url", narInfo->url); + jsonPath["url"] = narInfo->url; if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true)); + jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashBase, true); if (narInfo->fileSize) - jsonPath.attr("downloadSize", narInfo->fileSize); + jsonPath["downloadSize"] = narInfo->fileSize; if (showClosureSize) - jsonPath.attr("closureDownloadSize", closureSizes.second); + jsonPath["closureDownloadSize"] = closureSizes.second; } } } catch (InvalidPath &) { - jsonPath.attr("path", printStorePath(storePath)); - jsonPath.attr("valid", false); + jsonPath["path"] = printStorePath(storePath); + jsonPath["valid"] = false; } } + return jsonList; } @@ -897,7 +1048,9 @@ // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore.makeFixedOutputPathFromCA(info->path.name(), *info->ca); + info2->path = dstStore.makeFixedOutputPathFromCA( + info->path.name(), + info->contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -937,7 +1090,7 @@ for (auto & path : paths) { storePaths.insert(path.path()); if (auto realisation = std::get_if(&path.raw)) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); toplevelRealisations.insert(*realisation); } } @@ -992,113 +1145,72 @@ for (auto & path : storePaths) if (!valid.count(path)) missing.insert(path); - std::map pathsMap; - for (auto & path : storePaths) - pathsMap.insert_or_assign(path, path); - Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); - auto sorted = srcStore.topoSortPaths(missing); - std::reverse(sorted.begin(), sorted.end()); - - auto source = sinkToSource([&](Sink & sink) { - sink << sorted.size(); - for (auto & storePath : sorted) { - auto srcUri = srcStore.getUri(); - auto dstUri = dstStore.getUri(); - auto storePathS = srcStore.printStorePath(storePath); - Activity act(*logger, lvlInfo, actCopyPath, - makeCopyPathMessage(srcUri, dstUri, storePathS), - {storePathS, srcUri, dstUri}); - PushActivity pact(act.id); - - auto info = srcStore.queryPathInfo(storePath); - info->write(sink, srcStore, 16); - srcStore.narFromPath(storePath, sink); - } - }); - - dstStore.addMultipleToStore(*source, repair, checkSigs); + // In the general case, `addMultipleToStore` requires a sorted list of + // store paths to add, so sort them right now + auto sortedMissing = srcStore.topoSortPaths(missing); + std::reverse(sortedMissing.begin(), sortedMissing.end()); - #if 0 - std::atomic nrDone{0}; - std::atomic nrFailed{0}; - std::atomic bytesExpected{0}; - std::atomic nrRunning{0}; - - auto showProgress = [&]() { - act.progress(nrDone, missing.size(), nrRunning, nrFailed); - }; - - ThreadPool pool; + std::map pathsMap; + for (auto & path : storePaths) + pathsMap.insert_or_assign(path, path); - processGraph(pool, - StorePathSet(missing.begin(), missing.end()), + Store::PathsSource pathsToCopy; - [&](const StorePath & storePath) { - auto info = srcStore.queryPathInfo(storePath); - auto storePathForDst = storePath; - if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore.storeDir == srcStore.storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", - srcStore.printStorePath(storePath), + auto computeStorePathForDst = [&](const ValidPathInfo & currentPathInfo) -> StorePath { + auto storePathForSrc = currentPathInfo.path; + auto storePathForDst = storePathForSrc; + if (currentPathInfo.ca && currentPathInfo.references.empty()) { + storePathForDst = dstStore.makeFixedOutputPathFromCA( + currentPathInfo.path.name(), + currentPathInfo.contentAddressWithReferences().value()); + if (dstStore.storeDir == srcStore.storeDir) + assert(storePathForDst == storePathForSrc); + if (storePathForDst != storePathForSrc) + debug("replaced path '%s' to '%s' for substituter '%s'", + srcStore.printStorePath(storePathForSrc), dstStore.printStorePath(storePathForDst), dstStore.getUri()); - } - pathsMap.insert_or_assign(storePath, storePathForDst); - - if (dstStore.isValidPath(storePath)) { - nrDone++; - showProgress(); - return StorePathSet(); - } + } + return storePathForDst; + }; - bytesExpected += info->narSize; - act.setExpected(actCopyPath, bytesExpected); + // total is accessed by each copy, which are each handled in separate threads + std::atomic total = 0; - return info->references; - }, + for (auto & missingPath : sortedMissing) { + auto info = srcStore.queryPathInfo(missingPath); - [&](const StorePath & storePath) { - checkInterrupt(); + auto storePathForDst = computeStorePathForDst(*info); + pathsMap.insert_or_assign(missingPath, storePathForDst); - auto info = srcStore.queryPathInfo(storePath); + ValidPathInfo infoForDst = *info; + infoForDst.path = storePathForDst; - auto storePathForDst = storePath; - if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore.storeDir == srcStore.storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", - srcStore.printStorePath(storePath), - dstStore.printStorePath(storePathForDst), - dstStore.getUri()); - } - pathsMap.insert_or_assign(storePath, storePathForDst); + auto source = sinkToSource([&](Sink & sink) { + // We can reasonably assume that the copy will happen whenever we + // read the path, so log something about that at that point + auto srcUri = srcStore.getUri(); + auto dstUri = dstStore.getUri(); + auto storePathS = srcStore.printStorePath(missingPath); + Activity act(*logger, lvlInfo, actCopyPath, + makeCopyPathMessage(srcUri, dstUri, storePathS), + {storePathS, srcUri, dstUri}); + PushActivity pact(act.id); - if (!dstStore.isValidPath(storePathForDst)) { - MaintainCount mc(nrRunning); - showProgress(); - try { - copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); - } catch (Error &e) { - nrFailed++; - if (!settings.keepGoing) - throw e; - printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what()); - showProgress(); - return; - } - } + LambdaSink progressSink([&](std::string_view data) { + total += data.size(); + act.progress(total, info->narSize); + }); + TeeSink tee { sink, progressSink }; - nrDone++; - showProgress(); + srcStore.narFromPath(missingPath, tee); }); - #endif + pathsToCopy.push_back(std::pair{infoForDst, std::move(source)}); + } + + dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); return pathsMap; } @@ -1183,79 +1295,6 @@ } -std::string ValidPathInfo::fingerprint(const Store & store) const -{ - if (narSize == 0) - throw Error("cannot calculate fingerprint of path '%s' because its size is not known", - store.printStorePath(path)); - return - "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" - + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); -} - - -void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) -{ - sigs.insert(secretKey.signDetached(fingerprint(store))); -} - -bool ValidPathInfo::isContentAddressed(const Store & store) const -{ - if (! ca) return false; - - auto caPath = std::visit(overloaded { - [&](const TextHash & th) { - return store.makeTextPath(path.name(), th.hash, references); - }, - [&](const FixedOutputHash & fsh) { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); - } - }, *ca); - - bool res = caPath == path; - - if (!res) - printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); - - return res; -} - - -size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const -{ - if (isContentAddressed(store)) return maxSigs; - - size_t good = 0; - for (auto & sig : sigs) - if (checkSignature(store, publicKeys, sig)) - good++; - return good; -} - - -bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const -{ - return verifyDetached(fingerprint(store), sig, publicKeys); -} - - -Strings ValidPathInfo::shortRefs() const -{ - Strings refs; - for (auto & r : references) - refs.push_back(std::string(r.to_string())); - return refs; -} - - Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); @@ -1274,6 +1313,34 @@ } } +std::optional Store::getBuildDerivationPath(const StorePath & path) +{ + + if (!path.isDerivation()) { + try { + auto info = queryPathInfo(path); + if (!info->deriver) return std::nullopt; + return *info->deriver; + } catch (InvalidPath &) { + return std::nullopt; + } + } + + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations) || !isValidPath(path)) + return path; + + auto drv = readDerivation(path); + if (!drv.type().hasKnownOutputPaths()) { + // The build log is actually attached to the corresponding + // resolved derivation, so we need to get it first + auto resolvedDrv = drv.tryResolve(*this); + if (resolvedDrv) + return writeDerivation(*this, *resolvedDrv, NoRepair, true); + } + + return path; +} + Derivation Store::readDerivation(const StorePath & drvPath) { return readDerivationCommon(*this, drvPath, true); } @@ -1302,7 +1369,8 @@ return {uri, params}; } -static bool isNonUriPath(const std::string & spec) { +static bool isNonUriPath(const std::string & spec) +{ return // is not a URL spec.find("://") == std::string::npos @@ -1314,11 +1382,36 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Params & params) { if (uri == "" || uri == "auto") { - auto stateDir = get(params, "state").value_or(settings.nixStateDir); + auto stateDir = getOr(params, "state", settings.nixStateDir); if (access(stateDir.c_str(), R_OK | W_OK) == 0) return std::make_shared(params); else if (pathExists(settings.nixDaemonSocketFile)) return std::make_shared(params); + #if __linux__ + else if (!pathExists(stateDir) + && params.empty() + && getuid() != 0 + && !getEnv("NIX_STORE_DIR").has_value() + && !getEnv("NIX_STATE_DIR").has_value()) + { + /* If /nix doesn't exist, there is no daemon socket, and + we're not root, then automatically set up a chroot + store in ~/.local/share/nix/root. */ + auto chrootStore = getDataDir() + "/nix/root"; + if (!pathExists(chrootStore)) { + try { + createDirs(chrootStore); + } catch (Error & e) { + return std::make_shared(params); + } + warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + } else + debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + Store::Params params2; + params2["root"] = chrootStore; + return std::make_shared(params2); + } + #endif else return std::make_shared(params); } else if (uri == "daemon") { diff -Nru nix-2.8.0/src/libstore/store-api.hh nix-2.16.1+dfsg/src/libstore/store-api.hh --- nix-2.8.0/src/libstore/store-api.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/store-api.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,5 +1,7 @@ #pragma once +///@file +#include "nar-info.hh" #include "realisation.hh" #include "path.hh" #include "derived-path.hh" @@ -13,6 +15,7 @@ #include "path-info.hh" #include "repair-flag.hh" +#include #include #include #include @@ -53,7 +56,10 @@ */ MakeError(SubstError, Error); -MakeError(BuildError, Error); // denotes a permanent build failure +/** + * denotes a permanent build failure + */ +MakeError(BuildError, Error); MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); @@ -67,21 +73,29 @@ class FSAccessor; class NarInfoDiskCache; class Store; -class JSONPlaceholder; + + +typedef std::map OutputPathMap; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; -/* Magic header of exportPath() output (obsolete). */ +/** + * Magic header of exportPath() output (obsolete). + */ const uint32_t exportMagic = 0x4558494e; enum BuildMode { bmNormal, bmRepair, bmCheck }; +enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; struct BuildResult; +struct KeyedBuildResult; + +typedef std::map> StorePathCAMap; struct StoreConfig : public Config { @@ -95,17 +109,41 @@ virtual const std::string name() = 0; + virtual std::string doc() + { + return ""; + } + const PathSetting storeDir_{this, false, settings.nixStore, - "store", "path to the Nix store"}; + "store", + R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )"}; const Path storeDir = storeDir_; - const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; - - const Setting isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; + const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", + "Size of the in-memory store path metadata cache."}; - Setting priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; - - Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; + const Setting isTrusted{this, false, "trusted", + R"( + Whether paths from this store can be used as substitutes + even if they are not signed by a key listed in the + [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) + setting. + )"}; + + Setting priority{this, 0, "priority", + R"( + Priority of this store when used as a substituter. A lower value means a higher priority. + )"}; + + Setting wantMassQuery{this, false, "want-mass-query", + R"( + Whether this store (when used as a substituter) can be + queried efficiently for path validity. + )"}; Setting systemFeatures{this, getDefaultSystemFeatures(), "system-features", @@ -123,17 +161,26 @@ struct PathInfoCacheValue { - // Time of cache entry creation or update + /** + * Time of cache entry creation or update + */ std::chrono::time_point time_point = std::chrono::steady_clock::now(); - // Null if missing + /** + * Null if missing + */ std::shared_ptr value; - // Whether the value is valid as a cache entry. The path may not exist. + /** + * Whether the value is valid as a cache entry. The path may not + * exist. + */ bool isKnownNow(); - // Past tense, because a path can only be assumed to exists when - // isKnownNow() && didExist() + /** + * Past tense, because a path can only be assumed to exists when + * isKnownNow() && didExist() + */ inline bool didExist() { return value != nullptr; } @@ -167,35 +214,53 @@ std::string printStorePath(const StorePath & path) const; - // FIXME: remove + /** + * Deprecated + * + * \todo remove + */ StorePathSet parseStorePathSet(const PathSet & paths) const; PathSet printStorePathSet(const StorePathSet & path) const; - /* Display a set of paths in human-readable form (i.e., between quotes - and separated by commas). */ + /** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ std::string showPaths(const StorePathSet & paths); - /* Return true if ‘path’ is in the Nix store (but not the Nix - store itself). */ - bool isInStore(const Path & path) const; + /** + * @return true if ‘path’ is in the Nix store (but not the Nix + * store itself). + */ + bool isInStore(PathView path) const; - /* Return true if ‘path’ is a store path, i.e. a direct child of - the Nix store. */ + /** + * @return true if ‘path’ is a store path, i.e. a direct child of the + * Nix store. + */ bool isStorePath(std::string_view path) const; - /* Split a path like /nix/store/-/ into - /nix/store/- and /. */ - std::pair toStorePath(const Path & path) const; + /** + * Split a path like /nix/store/-/ into + * /nix/store/- and /. + */ + std::pair toStorePath(PathView path) const; - /* Follow symlinks until we end up with a path in the Nix store. */ + /** + * Follow symlinks until we end up with a path in the Nix store. + */ Path followLinksToStore(std::string_view path) const; - /* Same as followLinksToStore(), but apply toStorePath() to the - result. */ + /** + * Same as followLinksToStore(), but apply toStorePath() to the + * result. + */ StorePath followLinksToStorePath(std::string_view path) const; - /* Constructs a unique store path name. */ + /** + * Constructs a unique store path name. + */ StorePath makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const; StorePath makeStorePath(std::string_view type, @@ -204,45 +269,46 @@ StorePath makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(FileIngestionMethod method, - const Hash & hash, std::string_view name, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; - - StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references = {}) const; - - StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; - - /* This is the preparatory part of addToStore(); it computes the - store path to which srcPath is to be copied. Returns the store - path and the cryptographic hash of the contents of srcPath. */ + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; + + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; + + StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; + + /** + * Preparatory part of addToStore(). + * + * @return the store path to which srcPath is to be copied + * and the cryptographic hash of the contents of srcPath. + */ std::pair computeStorePathForPath(std::string_view name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; - /* Preparatory part of addTextToStore(). - - !!! Computation of the path should take the references given to - addTextToStore() into account, otherwise we have a (relatively - minor) security hole: a caller can register a source file with - bogus references. If there are too many references, the path may - not be garbage collected when it has to be (not really a problem, - the caller could create a root anyway), or it may be garbage - collected when it shouldn't be (more serious). - - Hashing the references would solve this (bogus references would - simply yield a different store path, so other users wouldn't be - affected), but it has some backwards compatibility issues (the - hashing scheme changes), so I'm not doing that for now. */ + /** + * Preparatory part of addTextToStore(). + * + * !!! Computation of the path should take the references given to + * addTextToStore() into account, otherwise we have a (relatively + * minor) security hole: a caller can register a source file with + * bogus references. If there are too many references, the path may + * not be garbage collected when it has to be (not really a problem, + * the caller could create a root anyway), or it may be garbage + * collected when it shouldn't be (more serious). + * + * Hashing the references would solve this (bogus references would + * simply yield a different store path, so other users wouldn't be + * affected), but it has some backwards compatibility issues (the + * hashing scheme changes), so I'm not doing that for now. + */ StorePath computeStorePathForText( std::string_view name, std::string_view s, const StorePathSet & references) const; - /* Check whether a path is valid. */ + /** + * Check whether a path is valid. + */ bool isValidPath(const StorePath & path); protected: @@ -251,53 +317,68 @@ public: - /* If requested, substitute missing paths. This - implements nix-copy-closure's --use-substitutes - flag. */ + /** + * If requested, substitute missing paths. This + * implements nix-copy-closure's --use-substitutes + * flag. + */ void substitutePaths(const StorePathSet & paths); - /* Query which of the given paths is valid. Optionally, try to - substitute missing paths. */ + /** + * Query which of the given paths is valid. Optionally, try to + * substitute missing paths. + */ virtual StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute); - /* Query the set of all valid paths. Note that for some store - backends, the name part of store paths may be replaced by 'x' - (i.e. you'll get /nix/store/-x rather than - /nix/store/-). Use queryPathInfo() to obtain the - full store path. FIXME: should return a set of - std::variant to get rid of this hack. */ + /** + * Query the set of all valid paths. Note that for some store + * backends, the name part of store paths may be replaced by 'x' + * (i.e. you'll get /nix/store/-x rather than + * /nix/store/-). Use queryPathInfo() to obtain the + * full store path. FIXME: should return a set of + * std::variant to get rid of this hack. + */ virtual StorePathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } constexpr static const char * MissingName = "x"; - /* Query information about a valid path. It is permitted to omit - the name part of the store path. */ + /** + * Query information about a valid path. It is permitted to omit + * the name part of the store path. + */ ref queryPathInfo(const StorePath & path); - /* Asynchronous version of queryPathInfo(). */ + /** + * Asynchronous version of queryPathInfo(). + */ void queryPathInfo(const StorePath & path, Callback> callback) noexcept; - /* Query the information about a realisation. */ + /** + * Query the information about a realisation. + */ std::shared_ptr queryRealisation(const DrvOutput &); - /* Asynchronous version of queryRealisation(). */ + /** + * Asynchronous version of queryRealisation(). + */ void queryRealisation(const DrvOutput &, Callback> callback) noexcept; - /* Check whether the given valid path info is sufficiently attested, by - either being signed by a trusted public key or content-addressed, in - order to be included in the given store. - - These same checks would be performed in addToStore, but this allows an - earlier failure in the case where dependencies need to be added too, but - the addToStore wouldn't fail until those dependencies are added. Also, - we don't really want to add the dependencies listed in a nar info we - don't trust anyyways. - */ + /** + * Check whether the given valid path info is sufficiently attested, by + * either being signed by a trusted public key or content-addressed, in + * order to be included in the given store. + * + * These same checks would be performed in addToStore, but this allows an + * earlier failure in the case where dependencies need to be added too, but + * the addToStore wouldn't fail until those dependencies are added. Also, + * we don't really want to add the dependencies listed in a nar info we + * don't trust anyyways. + */ virtual bool pathInfoIsUntrusted(const ValidPathInfo &) { return true; @@ -317,58 +398,96 @@ public: - /* Queries the set of incoming FS references for a store path. - The result is not cleared. */ + /** + * Queries the set of incoming FS references for a store path. + * The result is not cleared. + */ virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) { unsupported("queryReferrers"); } - /* Return all currently valid derivations that have `path' as an - output. (Note that the result of `queryDeriver()' is the - derivation that was actually used to produce `path', which may - not exist anymore.) */ + /** + * @return all currently valid derivations that have `path` as an + * output. + * + * (Note that the result of `queryDeriver()` is the derivation that + * was actually used to produce `path`, which may not exist + * anymore.) + */ virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; - /* Query the outputs of the derivation denoted by `path'. */ + /** + * Query the outputs of the derivation denoted by `path`. + */ virtual StorePathSet queryDerivationOutputs(const StorePath & path); - /* Query the mapping outputName => outputPath for the given derivation. All - outputs are mentioned so ones mising the mapping are mapped to - `std::nullopt`. */ + /** + * Query the mapping outputName => outputPath for the given + * derivation. All outputs are mentioned so ones mising the mapping + * are mapped to `std::nullopt`. + */ virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); - /* Query the mapping outputName=>outputPath for the given derivation. - Assume every output has a mapping and throw an exception otherwise. */ + /** + * Query the mapping outputName=>outputPath for the given derivation. + * Assume every output has a mapping and throw an exception otherwise. + */ OutputPathMap queryDerivationOutputMap(const StorePath & path); - /* Query the full store path given the hash part of a valid store - path, or empty if the path doesn't exist. */ + /** + * Query the full store path given the hash part of a valid store + * path, or empty if the path doesn't exist. + */ virtual std::optional queryPathFromHashPart(const std::string & hashPart) = 0; - /* Query which of the given paths have substitutes. */ + /** + * Query which of the given paths have substitutes. + */ virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; - /* Query substitute info (i.e. references, derivers and download - sizes) of a map of paths to their optional ca values. The info - of the first succeeding substituter for each path will be - returned. If a path does not have substitute info, it's omitted - from the resulting ‘infos’ map. */ + /** + * Query substitute info (i.e. references, derivers and download + * sizes) of a map of paths to their optional ca values. The info of + * the first succeeding substituter for each path will be returned. + * If a path does not have substitute info, it's omitted from the + * resulting ‘infos’ map. + */ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) { return; }; + SubstitutablePathInfos & infos); - /* Import a path into the store. */ + /** + * Import a path into the store. + */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; - /* Import multiple paths into the store. */ + /** + * A list of paths infos along with a source providing the content + * of the associated store path + */ + using PathsSource = std::vector>>; + + /** + * Import multiple paths into the store. + */ virtual void addMultipleToStore( Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); - /* Copy the contents of a path to the store and register the - validity the resulting path. The resulting path is returned. - The function object `filter' can be used to exclude files (see - libutil/archive.hh). */ + virtual void addMultipleToStore( + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs); + + /** + * Copy the contents of a path to the store and register the + * validity the resulting path. + * + * @return The resulting path is returned. + * @param filter This function can be used to exclude files (see + * libutil/archive.hh). + */ virtual StorePath addToStore( std::string_view name, const Path & srcPath, @@ -378,26 +497,33 @@ RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()); - /* Copy the contents of a path to the store and register the - validity the resulting path, using a constant amount of - memory. */ + /** + * Copy the contents of a path to the store and register the + * validity the resulting path, using a constant amount of + * memory. + */ ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, std::optional expectedCAHash = {}); - /* Like addToStore(), but the contents of the path are contained - in `dump', which is either a NAR serialisation (if recursive == - true) or simply the contents of a regular file (if recursive == - false). - `dump` may be drained */ - // FIXME: remove? + /** + * Like addToStore(), but the contents of the path are contained + * in `dump`, which is either a NAR serialisation (if recursive == + * true) or simply the contents of a regular file (if recursive == + * false). + * `dump` may be drained + * + * \todo remove? + */ virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) { unsupported("addToStoreFromDump"); } - /* Like addToStore, but the contents written to the output path is - a regular file containing the given string. */ + /** + * Like addToStore, but the contents written to the output path is a + * regular file containing the given string. + */ virtual StorePath addTextToStore( std::string_view name, std::string_view s, @@ -418,140 +544,178 @@ virtual void registerDrvOutput(const Realisation & output, CheckSigsFlag checkSigs) { return registerDrvOutput(output); } - /* Write a NAR dump of a store path. */ + /** + * Write a NAR dump of a store path. + */ virtual void narFromPath(const StorePath & path, Sink & sink) = 0; - /* For each path, if it's a derivation, build it. Building a - derivation means ensuring that the output paths are valid. If - they are already valid, this is a no-op. Otherwise, validity - can be reached in two ways. First, if the output paths is - substitutable, then build the path that way. Second, the - output paths can be created by running the builder, after - recursively building any sub-derivations. For inputs that are - not derivations, substitute them. */ + /** + * For each path, if it's a derivation, build it. Building a + * derivation means ensuring that the output paths are valid. If + * they are already valid, this is a no-op. Otherwise, validity + * can be reached in two ways. First, if the output paths is + * substitutable, then build the path that way. Second, the + * output paths can be created by running the builder, after + * recursively building any sub-derivations. For inputs that are + * not derivations, substitute them. + */ virtual void buildPaths( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr); - /* Like `buildPaths()`, but return a vector of `BuildResult`s - corresponding to each element in `paths`. Note that in case of - a build/substitution error, this function won't throw an - exception, but return a `BuildResult` containing an error - message. */ - virtual std::vector buildPathsWithResults( + /** + * Like buildPaths(), but return a vector of \ref BuildResult + * BuildResults corresponding to each element in paths. Note that in + * case of a build/substitution error, this function won't throw an + * exception, but return a BuildResult containing an error message. + */ + virtual std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr); - /* Build a single non-materialized derivation (i.e. not from an - on-disk .drv file). - - ‘drvPath’ is used to deduplicate worker goals so it is imperative that - is correct. That said, it doesn't literally need to be store path that - would be calculated from writing this derivation to the store: it is OK - if it instead is that of a Derivation which would resolve to this (by - taking the outputs of it's input derivations and adding them as input - sources) such that the build time referenceable-paths are the same. - - In the input-addressed case, we usually *do* use an "original" - unresolved derivations's path, as that is what will be used in the - `buildPaths` case. Also, the input-addressed output paths are verified - only by that contents of that specific unresolved derivation, so it is - nice to keep that information around so if the original derivation is - ever obtained later, it can be verified whether the trusted user in fact - used the proper output path. - - In the content-addressed case, we want to always use the - resolved drv path calculated from the provided derivation. This serves - two purposes: - - - It keeps the operation trustless, by ruling out a maliciously - invalid drv path corresponding to a non-resolution-equivalent - derivation. - - - For the floating case in particular, it ensures that the derivation - to output mapping respects the resolution equivalence relation, so - one cannot choose different resolution-equivalent derivations to - subvert dependency coherence (i.e. the property that one doesn't end - up with multiple different versions of dependencies without - explicitly choosing to allow it). - */ + /** + * Build a single non-materialized derivation (i.e. not from an + * on-disk .drv file). + * + * @param drvPath This is used to deduplicate worker goals so it is + * imperative that is correct. That said, it doesn't literally need + * to be store path that would be calculated from writing this + * derivation to the store: it is OK if it instead is that of a + * Derivation which would resolve to this (by taking the outputs of + * it's input derivations and adding them as input sources) such + * that the build time referenceable-paths are the same. + * + * In the input-addressed case, we usually *do* use an "original" + * unresolved derivations's path, as that is what will be used in the + * buildPaths case. Also, the input-addressed output paths are verified + * only by that contents of that specific unresolved derivation, so it is + * nice to keep that information around so if the original derivation is + * ever obtained later, it can be verified whether the trusted user in fact + * used the proper output path. + * + * In the content-addressed case, we want to always use the resolved + * drv path calculated from the provided derivation. This serves two + * purposes: + * + * - It keeps the operation trustless, by ruling out a maliciously + * invalid drv path corresponding to a non-resolution-equivalent + * derivation. + * + * - For the floating case in particular, it ensures that the derivation + * to output mapping respects the resolution equivalence relation, so + * one cannot choose different resolution-equivalent derivations to + * subvert dependency coherence (i.e. the property that one doesn't end + * up with multiple different versions of dependencies without + * explicitly choosing to allow it). + */ virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - /* Ensure that a path is valid. If it is not currently valid, it - may be made valid by running a substitute (if defined for the - path). */ + /** + * Ensure that a path is valid. If it is not currently valid, it + * may be made valid by running a substitute (if defined for the + * path). + */ virtual void ensurePath(const StorePath & path); - /* Add a store path as a temporary root of the garbage collector. - The root disappears as soon as we exit. */ + /** + * Add a store path as a temporary root of the garbage collector. + * The root disappears as soon as we exit. + */ virtual void addTempRoot(const StorePath & path) { debug("not creating temporary root, store doesn't support GC"); } - /* Return a string representing information about the path that - can be loaded into the database using `nix-store --load-db' or - `nix-store --register-validity'. */ + /** + * @return a string representing information about the path that + * can be loaded into the database using `nix-store --load-db` or + * `nix-store --register-validity`. + */ std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); - /* Write a JSON representation of store path metadata, such as the - hash and the references. If ‘includeImpureInfo’ is true, - variable elements such as the registration time are - included. If ‘showClosureSize’ is true, the closure size of - each path is included. */ - void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, + /** + * Write a JSON representation of store path metadata, such as the + * hash and the references. + * + * @param includeImpureInfo If true, variable elements such as the + * registration time are included. + * + * @param showClosureSize If true, the closure size of each path is + * included. + */ + nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, Base hashBase = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); - /* Return the size of the closure of the specified path, that is, - the sum of the size of the NAR serialisation of each path in - the closure. */ + /** + * @return the size of the closure of the specified path, that is, + * the sum of the size of the NAR serialisation of each path in the + * closure. + */ std::pair getClosureSize(const StorePath & storePath); - /* Optimise the disk space usage of the Nix store by hard-linking files - with the same contents. */ + /** + * Optimise the disk space usage of the Nix store by hard-linking files + * with the same contents. + */ virtual void optimiseStore() { }; - /* Check the integrity of the Nix store. Returns true if errors - remain. */ + /** + * Check the integrity of the Nix store. + * + * @return true if errors remain. + */ virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; }; - /* Return an object to access files in the Nix store. */ - virtual ref getFSAccessor() - { unsupported("getFSAccessor"); } - - /* Repair the contents of the given path by redownloading it using - a substituter (if available). */ - virtual void repairPath(const StorePath & path) - { unsupported("repairPath"); } + /** + * @return An object to access files in the Nix store. + */ + virtual ref getFSAccessor() = 0; + + /** + * Repair the contents of the given path by redownloading it using + * a substituter (if available). + */ + virtual void repairPath(const StorePath & path); - /* Add signatures to the specified store path. The signatures are - not verified. */ + /** + * Add signatures to the specified store path. The signatures are + * not verified. + */ virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) { unsupported("addSignatures"); } /* Utility functions. */ - /* Read a derivation, after ensuring its existence through - ensurePath(). */ + /** + * Read a derivation, after ensuring its existence through + * ensurePath(). + */ Derivation derivationFromPath(const StorePath & drvPath); - /* Read a derivation (which must already be valid). */ + /** + * Read a derivation (which must already be valid). + */ Derivation readDerivation(const StorePath & drvPath); - /* Read a derivation from a potentially invalid path. */ + /** + * Read a derivation from a potentially invalid path. + */ Derivation readInvalidDerivation(const StorePath & drvPath); - /* Place in `out' the set of all store paths in the file system - closure of `storePath'; that is, all paths than can be directly - or indirectly reached from it. `out' is not cleared. If - `flipDirection' is true, the set of paths that can reach - `storePath' is returned; that is, the closures under the - `referrers' relation instead of the `references' relation is - returned. */ + /** + * @param [out] out Place in here the set of all store paths in the + * file system closure of `storePath`; that is, all paths than can + * be directly or indirectly reached from it. `out` is not cleared. + * + * @param flipDirection If true, the set of paths that can reach + * `storePath` is returned; that is, the closures under the + * `referrers` relation instead of the `references` relation is + * returned. + */ virtual void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); @@ -560,27 +724,34 @@ StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); - /* Given a set of paths that are to be built, return the set of - derivations that will be built, and the set of output paths - that will be substituted. */ + /** + * Given a set of paths that are to be built, return the set of + * derivations that will be built, and the set of output paths that + * will be substituted. + */ virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize); - /* Sort a set of paths topologically under the references - relation. If p refers to q, then p precedes q in this list. */ + /** + * Sort a set of paths topologically under the references + * relation. If p refers to q, then p precedes q in this list. + */ StorePaths topoSortPaths(const StorePathSet & paths); - /* Export multiple paths in the format expected by ‘nix-store - --import’. */ + /** + * Export multiple paths in the format expected by ‘nix-store + * --import’. + */ void exportPaths(const StorePathSet & paths, Sink & sink); void exportPath(const StorePath & path, Sink & sink); - /* Import a sequence of NAR dumps created by exportPaths() into - the Nix store. Optionally, the contents of the NARs are - preloaded into the specified FS accessor to speed up subsequent - access. */ + /** + * Import a sequence of NAR dumps created by exportPaths() into the + * Nix store. Optionally, the contents of the NARs are preloaded + * into the specified FS accessor to speed up subsequent access. + */ StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs); struct Stats @@ -602,28 +773,53 @@ const Stats & getStats(); - /* Computes the full closure of of a set of store-paths for e.g. - derivations that need this information for `exportReferencesGraph`. + /** + * Computes the full closure of of a set of store-paths for e.g. + * derivations that need this information for `exportReferencesGraph`. */ StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); - /* Hack to allow long-running processes like hydra-queue-runner to - occasionally flush their path info cache. */ + /** + * Given a store path, return the realisation actually used in the realisation of this path: + * - If the path is a content-addressed derivation, try to resolve it + * - Otherwise, find one of its derivers + */ + std::optional getBuildDerivationPath(const StorePath &); + + /** + * Hack to allow long-running processes like hydra-queue-runner to + * occasionally flush their path info cache. + */ void clearPathInfoCache() { state.lock()->pathInfoCache.clear(); } - /* Establish a connection to the store, for store types that have - a notion of connection. Otherwise this is a no-op. */ + /** + * Establish a connection to the store, for store types that have + * a notion of connection. Otherwise this is a no-op. + */ virtual void connect() { }; - /* Get the protocol version of this store or it's connection. */ + /** + * Get the protocol version of this store or it's connection. + */ virtual unsigned int getProtocol() { return 0; }; + /** + * @return/ whether store trusts *us*. + * + * `std::nullopt` means we do not know. + * + * @note This is the opposite of the StoreConfig::isTrusted + * store setting. That is about whether *we* trust the store. + */ + virtual std::optional isTrustedClient() = 0; + + virtual Path toRealPath(const Path & storePath) { return storePath; @@ -634,10 +830,7 @@ return toRealPath(printStorePath(storePath)); } - virtual void createUser(const std::string & userName, uid_t userId) - { } - - /* + /** * Synchronises the options of the client with those of the daemon * (a no-op when there’s no daemon) */ @@ -649,7 +842,13 @@ Stats stats; - /* Unsupported methods. */ + /** + * Helper for methods that are not unsupported: this is used for + * default definitions for virtual methods that are meant to be overriden. + * + * \todo Using this should be a last resort. It is better to make + * the method "virtual pure" and/or move it to a subclass. + */ [[noreturn]] void unsupported(const std::string & op) { throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri()); @@ -658,7 +857,9 @@ }; -/* Copy a path from one store to another. */ +/** + * Copy a path from one store to another. + */ void copyStorePath( Store & srcStore, Store & dstStore, @@ -667,12 +868,14 @@ CheckSigsFlag checkSigs = CheckSigs); -/* Copy store paths from one store to another. The paths may be copied - in parallel. They are copied in a topologically sorted order (i.e. - if A is a reference of B, then A is copied before B), but the set - of store paths is not automatically closed; use copyClosure() for - that. Returns a map of what each path was copied to the dstStore - as. */ +/** + * Copy store paths from one store to another. The paths may be copied + * in parallel. They are copied in a topologically sorted order (i.e. if + * A is a reference of B, then A is copied before B), but the set of + * store paths is not automatically closed; use copyClosure() for that. + * + * @return a map of what each path was copied to the dstStore as. + */ std::map copyPaths( Store & srcStore, Store & dstStore, const RealisedPath::Set &, @@ -687,7 +890,9 @@ CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -/* Copy the closure of `paths` from `srcStore` to `dstStore`. */ +/** + * Copy the closure of `paths` from `srcStore` to `dstStore`. + */ void copyClosure( Store & srcStore, Store & dstStore, const RealisedPath::Set & paths, @@ -702,47 +907,61 @@ CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -/* Remove the temporary roots file for this process. Any temporary - root becomes garbage after this point unless it has been registered - as a (permanent) root. */ +/** + * Remove the temporary roots file for this process. Any temporary + * root becomes garbage after this point unless it has been registered + * as a (permanent) root. + */ void removeTempRoots(); -/* Return a Store object to access the Nix store denoted by - ‘uri’ (slight misnomer...). Supported values are: - - * ‘local’: The Nix store in /nix/store and database in - /nix/var/nix/db, accessed directly. - - * ‘daemon’: The Nix store accessed via a Unix domain socket - connection to nix-daemon. - - * ‘unix://’: The Nix store accessed via a Unix domain socket - connection to nix-daemon, with the socket located at . - - * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on - whether the user has write access to the local Nix - store/database. - - * ‘file://’: A binary cache stored in . - - * ‘https://’: A binary cache accessed via HTTP. - - * ‘s3://’: A writable binary cache stored on Amazon's Simple - Storage Service. +/** + * Resolve the derived path completely, failing if any derivation output + * is unknown. + */ +OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); - * ‘ssh://[user@]’: A remote Nix store accessed by running - ‘nix-store --serve’ via SSH. - You can pass parameters to the store implementation by appending - ‘?key=value&key=value&...’ to the URI. -*/ +/** + * @return a Store object to access the Nix store denoted by + * ‘uri’ (slight misnomer...). + * + * @param uri Supported values are: + * + * - ‘local’: The Nix store in /nix/store and database in + * /nix/var/nix/db, accessed directly. + * + * - ‘daemon’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon. + * + * - ‘unix://’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon, with the socket located at . + * + * - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + * whether the user has write access to the local Nix + * store/database. + * + * - ‘file://’: A binary cache stored in . + * + * - ‘https://’: A binary cache accessed via HTTP. + * + * - ‘s3://’: A writable binary cache stored on Amazon's Simple + * Storage Service. + * + * - ‘ssh://[user@]’: A remote Nix store accessed by running + * ‘nix-store --serve’ via SSH. + * + * You can pass parameters to the store implementation by appending + * ‘?key=value&key=value&...’ to the URI. + */ ref openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); -/* Return the default substituter stores, defined by the - ‘substituters’ option and various legacy options. */ +/** + * @return the default substituter stores, defined by the + * ‘substituters’ option and various legacy options. + */ std::list> getDefaultSubstituters(); struct StoreFactory @@ -785,8 +1004,10 @@ }; -/* Display a set of paths in human-readable form (i.e., between quotes - and separated by commas). */ +/** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ std::string showPaths(const PathSet & paths); @@ -795,10 +1016,12 @@ std::istream & str, std::optional hashGiven = std::nullopt); -/* Split URI into protocol+hierarchy part and its parameter set. */ +/** + * Split URI into protocol+hierarchy part and its parameter set. + */ std::pair splitUriAndParams(const std::string & uri); -std::optional getDerivationCA(const BasicDerivation & drv); +const ContentAddress * getDerivationCA(const BasicDerivation & drv); std::map drvOutputReferences( Store & store, diff -Nru nix-2.8.0/src/libstore/store-cast.hh nix-2.16.1+dfsg/src/libstore/store-cast.hh --- nix-2.8.0/src/libstore/store-cast.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/store-cast.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,9 +1,17 @@ #pragma once +///@file #include "store-api.hh" namespace nix { +/** + * Helper to try downcasting a Store with a nice method if it fails. + * + * This is basically an alternative to the user-facing part of + * Store::unsupported that allows us to still have a nice message but + * better interface design. + */ template T & require(Store & store) { diff -Nru nix-2.8.0/src/libstore/tests/derivation.cc nix-2.16.1+dfsg/src/libstore/tests/derivation.cc --- nix-2.8.0/src/libstore/tests/derivation.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/tests/derivation.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,219 @@ +#include +#include + +#include "experimental-features.hh" +#include "derivations.hh" + +#include "tests/libstore.hh" + +namespace nix { + +class DerivationTest : public LibStoreTest +{ +public: + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; +}; + +class CaDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "ca-derivations"); + } +}; + +class DynDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + } +}; + +class ImpureDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "impure-derivations"); + } +}; + +#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (DerivationOutput { VAL }).toJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME)); \ + } \ + \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + DerivationOutput { VAL }, \ + DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + STR ## _json, \ + mockXpSettings)); \ + } + +TEST_JSON(DerivationTest, inputAddressed, + R"({ + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + })", + (DerivationOutput::InputAddressed { + .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), + }), + "drv-name", "output-name") + +TEST_JSON(DerivationTest, caFixedFlat, + R"({ + "hashAlgo": "sha256", + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" + })", + (DerivationOutput::CAFixed { + .ca = FixedOutputHash { + .method = FileIngestionMethod::Flat, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationTest, caFixedNAR, + R"({ + "hashAlgo": "r:sha256", + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + })", + (DerivationOutput::CAFixed { + .ca = FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(DynDerivationTest, caFixedText, + R"({ + "hashAlgo": "text:sha256", + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" + })", + (DerivationOutput::CAFixed { + .ca = TextHash { + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(CaDerivationTest, caFloating, + R"({ + "hashAlgo": "r:sha256" + })", + (DerivationOutput::CAFloating { + .method = FileIngestionMethod::Recursive, + .hashType = htSHA256, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationTest, deferred, + R"({ })", + DerivationOutput::Deferred { }, + "drv-name", "output-name") + +TEST_JSON(ImpureDerivationTest, impure, + R"({ + "hashAlgo": "r:sha256", + "impure": true + })", + (DerivationOutput::Impure { + .method = FileIngestionMethod::Recursive, + .hashType = htSHA256, + }), + "drv-name", "output-name") + +#undef TEST_JSON + +#define TEST_JSON(NAME, STR, VAL, DRV_NAME) \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (Derivation { VAL }).toJSON(*store)); \ + } \ + \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + Derivation { VAL }, \ + Derivation::fromJSON( \ + *store, \ + STR ## _json)); \ + } + +TEST_JSON(simple, + R"({ + "name": "my-derivation", + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": [ + "cat", + "dog" + ] + }, + "system": "wasm-sel4", + "builder": "foo", + "args": [ + "bar", + "baz" + ], + "env": { + "BIG_BAD": "WOLF" + }, + "outputs": {} + })", + ({ + Derivation drv; + drv.name = "my-derivation"; + drv.inputSrcs = { + store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + { + store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + "cat", + "dog", + }, + } + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + drv; + }), + "drv-name") + +#undef TEST_JSON + +} diff -Nru nix-2.8.0/src/libstore/tests/derived-path.cc nix-2.16.1+dfsg/src/libstore/tests/derived-path.cc --- nix-2.8.0/src/libstore/tests/derived-path.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/tests/derived-path.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,72 @@ +#include + +#include +#include +#include + +#include "tests/derived-path.hh" +#include "tests/libstore.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Opaque { + .path = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Built { + .drvPath = *gen::arbitrary(), + .outputs = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} + +namespace nix { + +class DerivedPathTest : public LibStoreTest +{ +}; + +// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is +// no a real fixture. +// +// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args +TEST_F(DerivedPathTest, force_init) +{ +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, + prop_legacy_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parseLegacy(*store, o.to_string_legacy(*store))); +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, + prop_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store))); +} + +} diff -Nru nix-2.8.0/src/libstore/tests/derived-path.hh nix-2.16.1+dfsg/src/libstore/tests/derived-path.hh --- nix-2.8.0/src/libstore/tests/derived-path.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/tests/derived-path.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,29 @@ +#pragma once +///@file + +#include + +#include + +#include "tests/path.hh" +#include "tests/outputs-spec.hh" + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff -Nru nix-2.8.0/src/libstore/tests/downstream-placeholder.cc nix-2.16.1+dfsg/src/libstore/tests/downstream-placeholder.cc --- nix-2.8.0/src/libstore/tests/downstream-placeholder.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/tests/downstream-placeholder.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,33 @@ +#include + +#include "downstream-placeholder.hh" + +namespace nix { + +TEST(DownstreamPlaceholder, unknownCaOutput) { + ASSERT_EQ( + DownstreamPlaceholder::unknownCaOutput( + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, + "out").render(), + "/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz"); +} + +TEST(DownstreamPlaceholder, unknownDerivation) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + ASSERT_EQ( + DownstreamPlaceholder::unknownDerivation( + DownstreamPlaceholder::unknownCaOutput( + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" }, + "out"), + "out", + mockXpSettings).render(), + "/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w"); +} + +} diff -Nru nix-2.8.0/src/libstore/tests/libstore.hh nix-2.16.1+dfsg/src/libstore/tests/libstore.hh --- nix-2.8.0/src/libstore/tests/libstore.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/tests/libstore.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,26 @@ +#pragma once +///@file + +#include +#include + +#include "store-api.hh" + +namespace nix { + +class LibStoreTest : public ::testing::Test { + public: + static void SetUpTestSuite() { + initLibStore(); + } + + protected: + LibStoreTest() + : store(openStore("dummy://")) + { } + + ref store; +}; + + +} /* namespace nix */ diff -Nru nix-2.8.0/src/libstore/tests/local.mk nix-2.16.1+dfsg/src/libstore/tests/local.mk --- nix-2.8.0/src/libstore/tests/local.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/tests/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,20 @@ -check: libstore-tests_RUN +check: libstore-tests-exe_RUN -programs += libstore-tests +programs += libstore-tests-exe + +libstore-tests-exe_NAME = libnixstore-tests + +libstore-tests-exe_DIR := $(d) + +libstore-tests-exe_INSTALL_DIR := + +libstore-tests-exe_LIBS = libstore-tests + +libstore-tests-exe_LDFLAGS := $(GTEST_LIBS) + +libraries += libstore-tests + +libstore-tests_NAME = libnixstore-tests libstore-tests_DIR := $(d) @@ -10,6 +24,6 @@ libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil -libstore-tests_LIBS = libstore libutil +libstore-tests_LIBS = libutil-tests libstore libutil -libstore-tests_LDFLAGS := $(GTEST_LIBS) +libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff -Nru nix-2.8.0/src/libstore/tests/nar-info-disk-cache.cc nix-2.16.1+dfsg/src/libstore/tests/nar-info-disk-cache.cc --- nix-2.8.0/src/libstore/tests/nar-info-disk-cache.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/tests/nar-info-disk-cache.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,123 @@ +#include "nar-info-disk-cache.hh" + +#include +#include +#include "sqlite.hh" +#include + + +namespace nix { + +TEST(NarInfoDiskCacheImpl, create_and_read) { + // This is a large single test to avoid some setup overhead. + + int prio = 12345; + bool wantMassQuery = true; + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir); + Path dbPath(tmpDir + "/test-narinfo-disk-cache.sqlite"); + + int savedId; + int barId; + SQLite db; + SQLiteStmt getIds; + + { + auto cache = getTestNarInfoDiskCache(dbPath); + + // Set up "background noise" and check that different caches receive different ids + { + auto bc1 = cache->createCache("https://bar", "/nix/storedir", wantMassQuery, prio); + auto bc2 = cache->createCache("https://xyz", "/nix/storedir", false, 12); + ASSERT_NE(bc1, bc2); + barId = bc1; + } + + // Check that the fields are saved and returned correctly. This does not test + // the select statement yet, because of in-memory caching. + savedId = cache->createCache("http://foo", "/nix/storedir", wantMassQuery, prio);; + { + auto r = cache->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(savedId, r->id); + } + + // We're going to pay special attention to the id field because we had a bug + // that changed it. + db = SQLite(dbPath); + getIds.create(db, "select id from BinaryCaches where url = 'http://foo'"); + + { + auto q(getIds.use()); + ASSERT_TRUE(q.next()); + ASSERT_EQ(savedId, q.getInt(0)); + ASSERT_FALSE(q.next()); + } + + // Pretend that the caches are older, but keep one up to date, as "background noise" + db.exec("update BinaryCaches set timestamp = timestamp - 1 - 7 * 24 * 3600 where url <> 'https://xyz';"); + + // This shows that the in-memory cache works + { + auto r = cache->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + } + } + + { + // We can't clear the in-memory cache, so we use a new cache object. This is + // more realistic anyway. + auto cache2 = getTestNarInfoDiskCache(dbPath); + + { + auto r = cache2->upToDateCacheExists("http://foo"); + ASSERT_FALSE(r); + } + + // "Update", same data, check that the id number is reused + cache2->createCache("http://foo", "/nix/storedir", wantMassQuery, prio); + + { + auto r = cache2->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->id, savedId); + } + + { + auto q(getIds.use()); + ASSERT_TRUE(q.next()); + auto currentId = q.getInt(0); + ASSERT_FALSE(q.next()); + ASSERT_EQ(currentId, savedId); + } + + // Check that the fields can be modified, and the id remains the same + { + auto r0 = cache2->upToDateCacheExists("https://bar"); + ASSERT_FALSE(r0); + + cache2->createCache("https://bar", "/nix/storedir", !wantMassQuery, prio + 10); + auto r = cache2->upToDateCacheExists("https://bar"); + ASSERT_EQ(r->wantMassQuery, !wantMassQuery); + ASSERT_EQ(r->priority, prio + 10); + ASSERT_EQ(r->id, barId); + } + + // // Force update (no use case yet; we only retrieve cache metadata when stale based on timestamp) + // { + // cache2->createCache("https://bar", "/nix/storedir", wantMassQuery, prio + 20); + // auto r = cache2->upToDateCacheExists("https://bar"); + // ASSERT_EQ(r->wantMassQuery, wantMassQuery); + // ASSERT_EQ(r->priority, prio + 20); + // } + } +} + +} diff -Nru nix-2.8.0/src/libstore/tests/outputs-spec.cc nix-2.16.1+dfsg/src/libstore/tests/outputs-spec.cc --- nix-2.8.0/src/libstore/tests/outputs-spec.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/tests/outputs-spec.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,235 @@ +#include "outputs-spec.hh" + +#include +#include +#include + +namespace nix { + +#ifndef NDEBUG +TEST(OutputsSpec, no_empty_names) { + ASSERT_DEATH(OutputsSpec::Names { std::set { } }, ""); +} +#endif + +#define TEST_DONT_PARSE(NAME, STR) \ + TEST(OutputsSpec, bad_ ## NAME) { \ + std::optional OutputsSpecOpt = \ + OutputsSpec::parseOpt(STR); \ + ASSERT_FALSE(OutputsSpecOpt); \ + } + +TEST_DONT_PARSE(empty, "") +TEST_DONT_PARSE(garbage, "&*()") +TEST_DONT_PARSE(double_star, "**") +TEST_DONT_PARSE(star_first, "*,foo") +TEST_DONT_PARSE(star_second, "foo,*") + +#undef TEST_DONT_PARSE + +TEST(OutputsSpec, all) { + std::string_view str = "*"; + OutputsSpec expected = OutputsSpec::All { }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_out) { + std::string_view str = "out"; + OutputsSpec expected = OutputsSpec::Names { "out" }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_underscore) { + std::string_view str = "a_b"; + OutputsSpec expected = OutputsSpec::Names { "a_b" }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_numberic) { + std::string_view str = "01"; + OutputsSpec expected = OutputsSpec::Names { "01" }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_out_bin) { + OutputsSpec expected = OutputsSpec::Names { "out", "bin" }; + ASSERT_EQ(OutputsSpec::parse("out,bin"), expected); + // N.B. This normalization is OK. + ASSERT_EQ(expected.to_string(), "bin,out"); +} + +#define TEST_SUBSET(X, THIS, THAT) \ + X((OutputsSpec { THIS }).isSubsetOf(THAT)); + +TEST(OutputsSpec, subsets_all_all) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::All { }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, subsets_names_all) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, subsets_names_names_eq) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::Names { "a" }); +} + +TEST(OutputsSpec, subsets_names_names_noneq) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, (OutputsSpec::Names { "a", "b" })); +} + +TEST(OutputsSpec, not_subsets_all_names) { + TEST_SUBSET(ASSERT_FALSE, OutputsSpec::All { }, OutputsSpec::Names { "a" }); +} + +TEST(OutputsSpec, not_subsets_names_names) { + TEST_SUBSET(ASSERT_FALSE, (OutputsSpec::Names { "a", "b" }), (OutputsSpec::Names { "a" })); +} + +#undef TEST_SUBSET + +#define TEST_UNION(RES, THIS, THAT) \ + ASSERT_EQ(OutputsSpec { RES }, (OutputsSpec { THIS }).union_(THAT)); + +TEST(OutputsSpec, union_all_all) { + TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, union_all_names) { + TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::Names { "a" }); +} + +TEST(OutputsSpec, union_names_all) { + TEST_UNION(OutputsSpec::All { }, OutputsSpec::Names { "a" }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, union_names_names) { + TEST_UNION((OutputsSpec::Names { "a", "b" }), OutputsSpec::Names { "a" }, OutputsSpec::Names { "b" }); +} + +#undef TEST_UNION + +#define TEST_DONT_PARSE(NAME, STR) \ + TEST(ExtendedOutputsSpec, bad_ ## NAME) { \ + std::optional extendedOutputsSpecOpt = \ + ExtendedOutputsSpec::parseOpt(STR); \ + ASSERT_FALSE(extendedOutputsSpecOpt); \ + } + +TEST_DONT_PARSE(carot_empty, "^") +TEST_DONT_PARSE(prefix_carot_empty, "foo^") +TEST_DONT_PARSE(garbage, "^&*()") +TEST_DONT_PARSE(double_star, "^**") +TEST_DONT_PARSE(star_first, "^*,foo") +TEST_DONT_PARSE(star_second, "^foo,*") + +#undef TEST_DONT_PARSE + +TEST(ExtendedOutputsSpec, defeault) { + std::string_view str = "foo"; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = ExtendedOutputsSpec::Default { }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), str); +} + +TEST(ExtendedOutputsSpec, all) { + std::string_view str = "foo^*"; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = OutputsSpec::All { }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), str); +} + +TEST(ExtendedOutputsSpec, out) { + std::string_view str = "foo^out"; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = OutputsSpec::Names { "out" }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), str); +} + +TEST(ExtendedOutputsSpec, out_bin) { + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^out,bin"); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bin,out"); +} + +TEST(ExtendedOutputsSpec, many_carrot) { + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^bar^out,bin"); + ASSERT_EQ(prefix, "foo^bar"); + ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bar^bin,out"); +} + + +#define TEST_JSON(TYPE, NAME, STR, VAL) \ + \ + TEST(TYPE, NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + ((nlohmann::json) TYPE { VAL })); \ + } \ + \ + TEST(TYPE, NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + TYPE { VAL }, \ + (STR ## _json).get()); \ + } + +TEST_JSON(OutputsSpec, all, R"(["*"])", OutputsSpec::All { }) +TEST_JSON(OutputsSpec, name, R"(["a"])", OutputsSpec::Names { "a" }) +TEST_JSON(OutputsSpec, names, R"(["a","b"])", (OutputsSpec::Names { "a", "b" })) + +TEST_JSON(ExtendedOutputsSpec, def, R"(null)", ExtendedOutputsSpec::Default { }) +TEST_JSON(ExtendedOutputsSpec, all, R"(["*"])", ExtendedOutputsSpec::Explicit { OutputsSpec::All { } }) +TEST_JSON(ExtendedOutputsSpec, name, R"(["a"])", ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a" } }) +TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a", "b" } })) + +#undef TEST_JSON + +} + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just((OutputsSpec) OutputsSpec::All { }); + case 1: + return gen::just((OutputsSpec) OutputsSpec::Names { + *gen::nonEmpty(gen::container(gen::map( + gen::arbitrary(), + [](StorePathName n) { return n.name; }))), + }); + default: + assert(false); + } +} + +} + +namespace nix { + +RC_GTEST_PROP( + OutputsSpec, + prop_round_rip, + (const OutputsSpec & o)) +{ + RC_ASSERT(o == OutputsSpec::parse(o.to_string())); +} + +} diff -Nru nix-2.8.0/src/libstore/tests/outputs-spec.hh nix-2.16.1+dfsg/src/libstore/tests/outputs-spec.hh --- nix-2.8.0/src/libstore/tests/outputs-spec.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/tests/outputs-spec.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,18 @@ +#pragma once +///@file + +#include + +#include + +#include + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff -Nru nix-2.8.0/src/libstore/tests/path.cc nix-2.16.1+dfsg/src/libstore/tests/path.cc --- nix-2.8.0/src/libstore/tests/path.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/tests/path.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,153 @@ +#include + +#include +#include +#include + +#include "path-regex.hh" +#include "store-api.hh" + +#include "tests/hash.hh" +#include "tests/libstore.hh" +#include "tests/path.hh" + +namespace nix { + +#define STORE_DIR "/nix/store/" +#define HASH_PART "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q" + +class StorePathTest : public LibStoreTest +{ +}; + +static std::regex nameRegex { std::string { nameRegexStr } }; + +#define TEST_DONT_PARSE(NAME, STR) \ + TEST_F(StorePathTest, bad_ ## NAME) { \ + std::string_view str = \ + STORE_DIR HASH_PART "-" STR; \ + ASSERT_THROW( \ + store->parseStorePath(str), \ + BadStorePath); \ + std::string name { STR }; \ + EXPECT_FALSE(std::regex_match(name, nameRegex)); \ + } + +TEST_DONT_PARSE(empty, "") +TEST_DONT_PARSE(garbage, "&*()") +TEST_DONT_PARSE(double_star, "**") +TEST_DONT_PARSE(star_first, "*,foo") +TEST_DONT_PARSE(star_second, "foo,*") +TEST_DONT_PARSE(bang, "foo!o") + +#undef TEST_DONT_PARSE + +#define TEST_DO_PARSE(NAME, STR) \ + TEST_F(StorePathTest, good_ ## NAME) { \ + std::string_view str = \ + STORE_DIR HASH_PART "-" STR; \ + auto p = store->parseStorePath(str); \ + std::string name { p.name() }; \ + EXPECT_TRUE(std::regex_match(name, nameRegex)); \ + } + +// 0-9 a-z A-Z + - . _ ? = + +TEST_DO_PARSE(numbers, "02345") +TEST_DO_PARSE(lower_case, "foo") +TEST_DO_PARSE(upper_case, "FOO") +TEST_DO_PARSE(plus, "foo+bar") +TEST_DO_PARSE(dash, "foo-dev") +TEST_DO_PARSE(underscore, "foo_bar") +TEST_DO_PARSE(period, "foo.txt") +TEST_DO_PARSE(question_mark, "foo?why") +TEST_DO_PARSE(equals_sign, "foo=foo") + +#undef TEST_DO_PARSE + +// For rapidcheck +void showValue(const StorePath & p, std::ostream & os) { + os << p.to_string(); +} + +} + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + auto len = *gen::inRange( + 1, + StorePath::MaxPathLen - std::string_view { HASH_PART }.size()); + + std::string pre; + pre.reserve(len); + + for (size_t c = 0; c < len; ++c) { + switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { + case 0 ... 9: + pre += '0' + i; + case 10 ... 35: + pre += 'A' + (i - 10); + break; + case 36 ... 61: + pre += 'a' + (i - 36); + break; + case 62: + pre += '+'; + break; + case 63: + pre += '-'; + break; + case 64: + pre += '.'; + break; + case 65: + pre += '_'; + break; + case 66: + pre += '?'; + break; + case 67: + pre += '='; + break; + default: + assert(false); + } + } + + return gen::just(StorePathName { + .name = std::move(pre), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(StorePath { + *gen::arbitrary(), + (*gen::arbitrary()).name, + }); +} + +} // namespace rc + +namespace nix { + +RC_GTEST_FIXTURE_PROP( + StorePathTest, + prop_regex_accept, + (const StorePath & p)) +{ + RC_ASSERT(std::regex_match(std::string { p.name() }, nameRegex)); +} + +RC_GTEST_FIXTURE_PROP( + StorePathTest, + prop_round_rip, + (const StorePath & p)) +{ + RC_ASSERT(p == store->parseStorePath(store->printStorePath(p))); +} + +} diff -Nru nix-2.8.0/src/libstore/tests/path.hh nix-2.16.1+dfsg/src/libstore/tests/path.hh --- nix-2.8.0/src/libstore/tests/path.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/tests/path.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,29 @@ +#pragma once +///@file + +#include + +#include + +namespace nix { + +struct StorePathName { + std::string name; +}; + +} + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff -Nru nix-2.8.0/src/libstore/uds-remote-store.cc nix-2.16.1+dfsg/src/libstore/uds-remote-store.cc --- nix-2.8.0/src/libstore/uds-remote-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/uds-remote-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -26,9 +26,9 @@ UDSRemoteStore::UDSRemoteStore( - const std::string scheme, - std::string socket_path, - const Params & params) + const std::string scheme, + std::string socket_path, + const Params & params) : UDSRemoteStore(params) { path.emplace(socket_path); diff -Nru nix-2.8.0/src/libstore/uds-remote-store.hh nix-2.16.1+dfsg/src/libstore/uds-remote-store.hh --- nix-2.8.0/src/libstore/uds-remote-store.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/uds-remote-store.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "remote-store.hh" #include "local-fs-store.hh" @@ -15,6 +16,13 @@ } const std::string name() override { return "Local Daemon Store"; } + + std::string doc() override + { + return + #include "uds-remote-store.md" + ; + } }; class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore @@ -29,9 +37,6 @@ static std::set uriSchemes() { return {"unix"}; } - bool sameMachine() override - { return true; } - ref getFSAccessor() override { return LocalFSStore::getFSAccessor(); } diff -Nru nix-2.8.0/src/libstore/uds-remote-store.md nix-2.16.1+dfsg/src/libstore/uds-remote-store.md --- nix-2.8.0/src/libstore/uds-remote-store.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/uds-remote-store.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,9 @@ +R"( + +**Store URL format**: `daemon`, `unix://`*path* + +This store type accesses a Nix store by talking to a Nix daemon +listening on the Unix domain socket *path*. The store pseudo-URL +`daemon` is equivalent to `unix:///nix/var/nix/daemon-socket/socket`. + +)" diff -Nru nix-2.8.0/src/libstore/worker-protocol.cc nix-2.16.1+dfsg/src/libstore/worker-protocol.cc --- nix-2.8.0/src/libstore/worker-protocol.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/worker-protocol.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,192 @@ +#include "serialise.hh" +#include "util.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "build-result.hh" +#include "worker-protocol.hh" +#include "archive.hh" +#include "derivations.hh" + +#include + +namespace nix { + +std::string WorkerProto::read(const Store & store, Source & from) +{ + return readString(from); +} + +void WorkerProto::write(const Store & store, Sink & out, const std::string & str) +{ + out << str; +} + + +StorePath WorkerProto::read(const Store & store, Source & from) +{ + return store.parseStorePath(readString(from)); +} + +void WorkerProto::write(const Store & store, Sink & out, const StorePath & storePath) +{ + out << store.printStorePath(storePath); +} + + +std::optional WorkerProto>::read(const Store & store, Source & from) +{ + auto temp = readNum(from); + switch (temp) { + case 0: + return std::nullopt; + case 1: + return { Trusted }; + case 2: + return { NotTrusted }; + default: + throw Error("Invalid trusted status from remote"); + } +} + +void WorkerProto>::write(const Store & store, Sink & out, const std::optional & optTrusted) +{ + if (!optTrusted) + out << (uint8_t)0; + else { + switch (*optTrusted) { + case Trusted: + out << (uint8_t)1; + break; + case NotTrusted: + out << (uint8_t)2; + break; + default: + assert(false); + }; + } +} + + +ContentAddress WorkerProto::read(const Store & store, Source & from) +{ + return ContentAddress::parse(readString(from)); +} + +void WorkerProto::write(const Store & store, Sink & out, const ContentAddress & ca) +{ + out << renderContentAddress(ca); +} + + +DerivedPath WorkerProto::read(const Store & store, Source & from) +{ + auto s = readString(from); + return DerivedPath::parseLegacy(store, s); +} + +void WorkerProto::write(const Store & store, Sink & out, const DerivedPath & req) +{ + out << req.to_string_legacy(store); +} + + +Realisation WorkerProto::read(const Store & store, Source & from) +{ + std::string rawInput = readString(from); + return Realisation::fromJSON( + nlohmann::json::parse(rawInput), + "remote-protocol" + ); +} + +void WorkerProto::write(const Store & store, Sink & out, const Realisation & realisation) +{ + out << realisation.toJSON().dump(); +} + + +DrvOutput WorkerProto::read(const Store & store, Source & from) +{ + return DrvOutput::parse(readString(from)); +} + +void WorkerProto::write(const Store & store, Sink & out, const DrvOutput & drvOutput) +{ + out << drvOutput.to_string(); +} + + +KeyedBuildResult WorkerProto::read(const Store & store, Source & from) +{ + auto path = WorkerProto::read(store, from); + auto br = WorkerProto::read(store, from); + return KeyedBuildResult { + std::move(br), + /* .path = */ std::move(path), + }; +} + +void WorkerProto::write(const Store & store, Sink & to, const KeyedBuildResult & res) +{ + workerProtoWrite(store, to, res.path); + workerProtoWrite(store, to, static_cast(res)); +} + + +BuildResult WorkerProto::read(const Store & store, Source & from) +{ + BuildResult res; + res.status = (BuildResult::Status) readInt(from); + from + >> res.errorMsg + >> res.timesBuilt + >> res.isNonDeterministic + >> res.startTime + >> res.stopTime; + auto builtOutputs = WorkerProto::read(store, from); + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + return res; +} + +void WorkerProto::write(const Store & store, Sink & to, const BuildResult & res) +{ + to + << res.status + << res.errorMsg + << res.timesBuilt + << res.isNonDeterministic + << res.startTime + << res.stopTime; + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + workerProtoWrite(store, to, builtOutputs); +} + + +std::optional WorkerProto>::read(const Store & store, Source & from) +{ + auto s = readString(from); + return s == "" ? std::optional {} : store.parseStorePath(s); +} + +void WorkerProto>::write(const Store & store, Sink & out, const std::optional & storePathOpt) +{ + out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + + +std::optional WorkerProto>::read(const Store & store, Source & from) +{ + return ContentAddress::parseOpt(readString(from)); +} + +void WorkerProto>::write(const Store & store, Sink & out, const std::optional & caOpt) +{ + out << (caOpt ? renderContentAddress(*caOpt) : ""); +} + +} diff -Nru nix-2.8.0/src/libstore/worker-protocol.hh nix-2.16.1+dfsg/src/libstore/worker-protocol.hh --- nix-2.8.0/src/libstore/worker-protocol.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libstore/worker-protocol.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,6 @@ #pragma once +///@file -#include "store-api.hh" #include "serialise.hh" namespace nix { @@ -9,11 +9,15 @@ #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 34) +#define PROTOCOL_VERSION (1 << 8 | 35) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) +/** + * Enumeration of all the request types for the "worker protocol", used + * by unix:// and ssh-ng:// stores. + */ typedef enum { wopIsValidPath = 1, wopHasSubstitutes = 3, @@ -74,114 +78,162 @@ class Store; struct Source; -/* To guide overloading */ -template -struct Phantom {}; - +// items being serialized +struct DerivedPath; +struct DrvOutput; +struct Realisation; +struct BuildResult; +struct KeyedBuildResult; +enum TrustedFlag : bool; -namespace worker_proto { -/* FIXME maybe move more stuff inside here */ -#define MAKE_WORKER_PROTO(TEMPLATE, T) \ - TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \ - TEMPLATE void write(const Store & store, Sink & out, const T & str) +/** + * Data type for canonical pairs of serializers for the worker protocol. + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ +template +struct WorkerProto { + static T read(const Store & store, Source & from); + static void write(const Store & store, Sink & out, const T & t); +}; + +/** + * Wrapper function around `WorkerProto::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ +template +void workerProtoWrite(const Store & store, Sink & out, const T & t) +{ + WorkerProto::write(store, out, t); +} -MAKE_WORKER_PROTO(, std::string); -MAKE_WORKER_PROTO(, StorePath); -MAKE_WORKER_PROTO(, ContentAddress); -MAKE_WORKER_PROTO(, DerivedPath); -MAKE_WORKER_PROTO(, Realisation); -MAKE_WORKER_PROTO(, DrvOutput); -MAKE_WORKER_PROTO(, BuildResult); +/** + * Declare a canonical serializer pair for the worker protocol. + * + * We specialize the struct merely to indicate that we are implementing + * the function for the given type. + * + * Some sort of `template<...>` must be used with the caller for this to + * be legal specialization syntax. See below for what that looks like in + * practice. + */ +#define MAKE_WORKER_PROTO(T) \ + struct WorkerProto< T > { \ + static T read(const Store & store, Source & from); \ + static void write(const Store & store, Sink & out, const T & t); \ + }; + +template<> +MAKE_WORKER_PROTO(std::string); +template<> +MAKE_WORKER_PROTO(StorePath); +template<> +MAKE_WORKER_PROTO(ContentAddress); +template<> +MAKE_WORKER_PROTO(DerivedPath); +template<> +MAKE_WORKER_PROTO(Realisation); +template<> +MAKE_WORKER_PROTO(DrvOutput); +template<> +MAKE_WORKER_PROTO(BuildResult); +template<> +MAKE_WORKER_PROTO(KeyedBuildResult); +template<> +MAKE_WORKER_PROTO(std::optional); -MAKE_WORKER_PROTO(template, std::vector); -MAKE_WORKER_PROTO(template, std::set); +template +MAKE_WORKER_PROTO(std::vector); +template +MAKE_WORKER_PROTO(std::set); -#define X_ template -#define Y_ std::map -MAKE_WORKER_PROTO(X_, Y_); +template +#define X_ std::map +MAKE_WORKER_PROTO(X_); #undef X_ -#undef Y_ - -/* These use the empty string for the null case, relying on the fact - that the underlying types never serialize to the empty string. - We do this instead of a generic std::optional instance because - ordinal tags (0 or 1, here) are a bit of a compatability hazard. For - the same reason, we don't have a std::variant instances (ordinal - tags 0...n). - - We could the generic instances and then these as specializations for - compatability, but that's proven a bit finnicky, and also makes the - worker protocol harder to implement in other languages where such - specializations may not be allowed. +/** + * These use the empty string for the null case, relying on the fact + * that the underlying types never serialize to the empty string. + * + * We do this instead of a generic std::optional instance because + * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + * the same reason, we don't have a std::variant instances (ordinal + * tags 0...n). + * + * We could the generic instances and then these as specializations for + * compatability, but that's proven a bit finnicky, and also makes the + * worker protocol harder to implement in other languages where such + * specializations may not be allowed. */ -MAKE_WORKER_PROTO(, std::optional); -MAKE_WORKER_PROTO(, std::optional); +template<> +MAKE_WORKER_PROTO(std::optional); +template<> +MAKE_WORKER_PROTO(std::optional); template -std::vector read(const Store & store, Source & from, Phantom> _) +std::vector WorkerProto>::read(const Store & store, Source & from) { std::vector resSet; auto size = readNum(from); while (size--) { - resSet.push_back(read(store, from, Phantom {})); + resSet.push_back(WorkerProto::read(store, from)); } return resSet; } template -void write(const Store & store, Sink & out, const std::vector & resSet) +void WorkerProto>::write(const Store & store, Sink & out, const std::vector & resSet) { out << resSet.size(); for (auto & key : resSet) { - write(store, out, key); + WorkerProto::write(store, out, key); } } template -std::set read(const Store & store, Source & from, Phantom> _) +std::set WorkerProto>::read(const Store & store, Source & from) { std::set resSet; auto size = readNum(from); while (size--) { - resSet.insert(read(store, from, Phantom {})); + resSet.insert(WorkerProto::read(store, from)); } return resSet; } template -void write(const Store & store, Sink & out, const std::set & resSet) +void WorkerProto>::write(const Store & store, Sink & out, const std::set & resSet) { out << resSet.size(); for (auto & key : resSet) { - write(store, out, key); + WorkerProto::write(store, out, key); } } template -std::map read(const Store & store, Source & from, Phantom> _) +std::map WorkerProto>::read(const Store & store, Source & from) { std::map resMap; auto size = readNum(from); while (size--) { - auto k = read(store, from, Phantom {}); - auto v = read(store, from, Phantom {}); + auto k = WorkerProto::read(store, from); + auto v = WorkerProto::read(store, from); resMap.insert_or_assign(std::move(k), std::move(v)); } return resMap; } template -void write(const Store & store, Sink & out, const std::map & resMap) +void WorkerProto>::write(const Store & store, Sink & out, const std::map & resMap) { out << resMap.size(); for (auto & i : resMap) { - write(store, out, i.first); - write(store, out, i.second); + WorkerProto::write(store, out, i.first); + WorkerProto::write(store, out, i.second); } } } - -} diff -Nru nix-2.8.0/src/libutil/abstract-setting-to-json.hh nix-2.16.1+dfsg/src/libutil/abstract-setting-to-json.hh --- nix-2.8.0/src/libutil/abstract-setting-to-json.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/abstract-setting-to-json.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include #include "config.hh" diff -Nru nix-2.8.0/src/libutil/ansicolor.hh nix-2.16.1+dfsg/src/libutil/ansicolor.hh --- nix-2.8.0/src/libutil/ansicolor.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/ansicolor.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,8 +1,12 @@ #pragma once +/** + * @file + * + * @brief Some ANSI escape sequences. + */ namespace nix { -/* Some ANSI escape sequences. */ #define ANSI_NORMAL "\e[0m" #define ANSI_BOLD "\e[1m" #define ANSI_FAINT "\e[2m" diff -Nru nix-2.8.0/src/libutil/archive.cc nix-2.16.1+dfsg/src/libutil/archive.cc --- nix-2.8.0/src/libutil/archive.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/archive.cc 2023-06-06 10:55:23.000000000 +0000 @@ -35,10 +35,6 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings); -const std::string narVersionMagic1 = "nix-archive-1"; - -static std::string caseHackSuffix = "~nix~case~hack~"; - PathFilter defaultPathFilter = [](const Path &) { return true; }; @@ -91,7 +87,7 @@ std::string name(i.name); size_t pos = i.name.find(caseHackSuffix); if (pos != std::string::npos) { - debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name)); + debug("removing case hack suffix from '%1%'", path + "/" + i.name); name.erase(pos); } if (!unhacked.emplace(name, i.name).second) @@ -234,6 +230,7 @@ else if (s == "contents" && type == tpRegular) { parseContents(sink, source, path); + sink.closeRegularFile(); } else if (s == "executable" && type == tpRegular) { @@ -265,7 +262,7 @@ if (archiveSettings.useCaseHack) { auto i = names.find(name); if (i != names.end()) { - debug(format("case collision between '%1%' and '%2%'") % i->first % name); + debug("case collision between '%1%' and '%2%'", i->first, name); name += caseHackSuffix; name += std::to_string(++i->second); } else @@ -324,6 +321,12 @@ if (!fd) throw SysError("creating file '%1%'", p); } + void closeRegularFile() override + { + /* Call close explicitly to make sure the error is checked */ + fd.close(); + } + void isExecutable() override { struct stat st; diff -Nru nix-2.8.0/src/libutil/archive.hh nix-2.16.1+dfsg/src/libutil/archive.hh --- nix-2.8.0/src/libutil/archive.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/archive.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "serialise.hh" @@ -7,59 +8,79 @@ namespace nix { -/* dumpPath creates a Nix archive of the specified path. The format - is as follows: - - IF path points to a REGULAR FILE: - dump(path) = attrs( - [ ("type", "regular") - , ("contents", contents(path)) - ]) - - IF path points to a DIRECTORY: - dump(path) = attrs( - [ ("type", "directory") - , ("entries", concat(map(f, sort(entries(path))))) - ]) - where f(fn) = attrs( - [ ("name", fn) - , ("file", dump(path + "/" + fn)) - ]) - - where: - - attrs(as) = concat(map(attr, as)) + encN(0) - attrs((a, b)) = encS(a) + encS(b) - - encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) - - encN(n) = 64-bit little-endian encoding of n. - - contents(path) = the contents of a regular file. - - sort(strings) = lexicographic sort by 8-bit value (strcmp). - - entries(path) = the entries of a directory, without `.' and - `..'. - - `+' denotes string concatenation. */ - - +/** + * dumpPath creates a Nix archive of the specified path. + * + * @param path the file system data to dump. Dumping is recursive so if + * this is a directory we dump it and all its children. + * + * @param [out] sink The serialised archive is fed into this sink. + * + * @param filter Can be used to skip certain files. + * + * The format is as follows: + * + * ``` + * IF path points to a REGULAR FILE: + * dump(path) = attrs( + * [ ("type", "regular") + * , ("contents", contents(path)) + * ]) + * + * IF path points to a DIRECTORY: + * dump(path) = attrs( + * [ ("type", "directory") + * , ("entries", concat(map(f, sort(entries(path))))) + * ]) + * where f(fn) = attrs( + * [ ("name", fn) + * , ("file", dump(path + "/" + fn)) + * ]) + * + * where: + * + * attrs(as) = concat(map(attr, as)) + encN(0) + * attrs((a, b)) = encS(a) + encS(b) + * + * encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) + * + * encN(n) = 64-bit little-endian encoding of n. + * + * contents(path) = the contents of a regular file. + * + * sort(strings) = lexicographic sort by 8-bit value (strcmp). + * + * entries(path) = the entries of a directory, without `.` and + * `..`. + * + * `+` denotes string concatenation. + * ``` + */ void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); -/* Same as `void dumpPath()`, but returns the last modified date of the path */ +/** + * Same as dumpPath(), but returns the last modified date of the path. + */ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +/** + * Dump an archive with a single file with these contents. + * + * @param s Contents of the file. + */ void dumpString(std::string_view s, Sink & sink); -/* FIXME: fix this API, it sucks. */ +/** + * \todo Fix this API, it sucks. + */ struct ParseSink { virtual void createDirectory(const Path & path) { }; virtual void createRegularFile(const Path & path) { }; + virtual void closeRegularFile() { }; virtual void isExecutable() { }; virtual void preallocateContents(uint64_t size) { }; virtual void receiveContents(std::string_view data) { }; @@ -67,8 +88,10 @@ virtual void createSymlink(const Path & path, const std::string & target) { }; }; -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ +/** + * If the NAR archive contains a single file at top-level, then save + * the contents of the file to `s`. Otherwise barf. + */ struct RetrieveRegularNARSink : ParseSink { bool regular = true; @@ -96,13 +119,17 @@ void restorePath(const Path & path, Source & source); -/* Read a NAR from 'source' and write it to 'sink'. */ +/** + * Read a NAR from 'source' and write it to 'sink'. + */ void copyNAR(Source & source, Sink & sink); void copyPath(const Path & from, const Path & to); -extern const std::string narVersionMagic1; +inline constexpr std::string_view narVersionMagic1 = "nix-archive-1"; + +inline constexpr std::string_view caseHackSuffix = "~nix~case~hack~"; } diff -Nru nix-2.8.0/src/libutil/args.cc nix-2.16.1+dfsg/src/libutil/args.cc --- nix-2.8.0/src/libutil/args.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/args.cc 2023-06-06 10:55:23.000000000 +0000 @@ -29,7 +29,15 @@ void Completions::add(std::string completion, std::string description) { - assert(description.find('\n') == std::string::npos); + description = trim(description); + // ellipsize overflowing content on the back of the description + auto end_index = description.find_first_of(".\n"); + if (end_index != std::string::npos) { + auto needs_ellipsis = end_index != description.size() - 1; + description.resize(end_index); + if (needs_ellipsis) + description.append(" [...]"); + } insert(Completion { .completion = completion, .description = description @@ -44,7 +52,7 @@ std::string completionMarker = "___COMPLETE___"; -std::optional needsCompletion(std::string_view s) +static std::optional needsCompletion(std::string_view s) { if (!completions) return {}; auto i = s.find(completionMarker); @@ -112,6 +120,12 @@ if (!argsSeen) initialFlagsProcessed(); + + /* Now that we are done parsing, make sure that any experimental + * feature required by the flags is enabled */ + for (auto & f : flagExperimentalFeatures) + experimentalFeatureSettings.require(f); + } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) @@ -120,12 +134,18 @@ auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; + + if (auto & f = flag.experimentalFeature) + flagExperimentalFeatures.insert(*f); + std::vector args; bool anyCompleted = false; for (size_t n = 0 ; n < flag.handler.arity; ++n) { if (pos == end) { - if (flag.handler.arity == ArityAny) break; - throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); + if (flag.handler.arity == ArityAny || anyCompleted) break; + throw UsageError( + "flag '%s' requires %d argument(s), but only %d were given", + name, flag.handler.arity, n); } if (auto prefix = needsCompletion(*pos)) { anyCompleted = true; @@ -144,7 +164,11 @@ for (auto & [name, flag] : longFlags) { if (!hiddenCategories.count(flag->category) && hasPrefix(name, std::string(*prefix, 2))) + { + if (auto & f = flag->experimentalFeature) + flagExperimentalFeatures.insert(*f); completions->add("--" + name, flag->description); + } } return false; } @@ -164,7 +188,8 @@ if (prefix == "-") { completions->add("--"); for (auto & [flagName, flag] : shortFlags) - completions->add(std::string("-") + flagName, flag->description); + if (experimentalFeatureSettings.isEnabled(flag->experimentalFeature)) + completions->add(std::string("-") + flagName, flag->description); } } @@ -216,12 +241,17 @@ if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); if (flag->description != "") - j["description"] = flag->description; + j["description"] = trim(flag->description); j["category"] = flag->category; if (flag->handler.arity != ArityAny) j["arity"] = flag->handler.arity; if (!flag->labels.empty()) j["labels"] = flag->labels; + // TODO With C++23 use `std::optional::tranform` + if (auto & xp = flag->experimentalFeature) + j["experimental-feature"] = showExperimentalFeature(*xp); + else + j["experimental-feature"] = nullptr; flags[name] = std::move(j); } @@ -237,7 +267,7 @@ } auto res = nlohmann::json::object(); - res["description"] = description(); + res["description"] = trim(description()); res["flags"] = std::move(flags); res["args"] = std::move(args); auto s = doc(); @@ -318,13 +348,18 @@ return args; } +std::optional Command::experimentalFeature () +{ + return { Xp::NixCommand }; +} + MultiCommand::MultiCommand(const Commands & commands_) : commands(commands_) { expectArgs({ .label = "subcommand", .optional = true, - .handler = {[=](std::string s) { + .handler = {[=,this](std::string s) { assert(!command); auto i = commands.find(s); if (i == commands.end()) { @@ -362,6 +397,14 @@ return Args::processArgs(args, finish); } +void MultiCommand::completionHook() +{ + if (command) + return command->second->completionHook(); + else + return Args::completionHook(); +} + nlohmann::json MultiCommand::toJSON() { auto cmds = nlohmann::json::object(); @@ -371,8 +414,13 @@ auto j = command->toJSON(); auto cat = nlohmann::json::object(); cat["id"] = command->category(); - cat["description"] = categories[command->category()]; + cat["description"] = trim(categories[command->category()]); j["category"] = std::move(cat); + // TODO With C++23 use `std::optional::tranform` + if (auto xp = command->experimentalFeature()) + cat["experimental-feature"] = showExperimentalFeature(*xp); + else + cat["experimental-feature"] = nullptr; cmds[name] = std::move(j); } diff -Nru nix-2.8.0/src/libutil/args.hh nix-2.16.1+dfsg/src/libutil/args.hh --- nix-2.8.0/src/libutil/args.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/args.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -18,14 +19,22 @@ { public: - /* Parse the command line, throwing a UsageError if something goes - wrong. */ + /** + * Parse the command line, throwing a UsageError if something goes + * wrong. + */ void parseCmdline(const Strings & cmdline); - /* Return a short one-line description of the command. */ + /** + * Return a short one-line description of the command. + */ virtual std::string description() { return ""; } - /* Return documentation about this command, in Markdown format. */ + virtual bool forceImpureByDefault() { return false; } + + /** + * Return documentation about this command, in Markdown format. + */ virtual std::string doc() { return ""; } protected: @@ -115,6 +124,8 @@ Handler handler; std::function completer; + std::optional experimentalFeature; + static Flag mkHashTypeFlag(std::string && longName, HashType * ht); static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oht); }; @@ -142,10 +153,19 @@ std::set hiddenCategories; - /* Called after all command line flags before the first non-flag - argument (if any) have been processed. */ + /** + * Called after all command line flags before the first non-flag + * argument (if any) have been processed. + */ virtual void initialFlagsProcessed() {} + /** + * Called after the command line has been processed if we need to generate + * completions. Useful for commands that need to know the whole command line + * in order to know what completions to generate. + */ + virtual void completionHook() { } + public: void addFlag(Flag && flag); @@ -157,7 +177,9 @@ expectedArgs.emplace_back(std::move(arg)); } - /* Expect a string argument. */ + /** + * Expect a string argument. + */ void expectArg(const std::string & label, std::string * dest, bool optional = false) { expectArgs({ @@ -167,7 +189,9 @@ }); } - /* Expect 0 or more arguments. */ + /** + * Expect 0 or more arguments. + */ void expectArgs(const std::string & label, std::vector * dest) { expectArgs({ @@ -181,30 +205,48 @@ friend class MultiCommand; MultiCommand * parent = nullptr; + +private: + + /** + * Experimental features needed when parsing args. These are checked + * after flag parsing is completed in order to support enabling + * experimental features coming after the flag that needs the + * experimental feature. + */ + std::set flagExperimentalFeatures; }; -/* A command is an argument parser that can be executed by calling its - run() method. */ +/** + * A command is an argument parser that can be executed by calling its + * run() method. + */ struct Command : virtual public Args { friend class MultiCommand; virtual ~Command() { } - virtual void prepare() { }; + /** + * Entry point to the command + */ virtual void run() = 0; typedef int Category; static constexpr Category catDefault = 0; + virtual std::optional experimentalFeature (); + virtual Category category() { return catDefault; } }; typedef std::map()>> Commands; -/* An argument parser that supports multiple subcommands, - i.e. ‘ ’. */ +/** + * An argument parser that supports multiple subcommands, + * i.e. ‘ ’. + */ class MultiCommand : virtual public Args { public: @@ -212,7 +254,9 @@ std::map categories; - // Selected command, if any. + /** + * Selected command, if any. + */ std::optional>> command; MultiCommand(const Commands & commands); @@ -221,6 +265,8 @@ bool processArgs(const Strings & args, bool finish) override; + void completionHook() override; + nlohmann::json toJSON() override; }; @@ -245,8 +291,6 @@ }; extern CompletionType completionType; -std::optional needsCompletion(std::string_view s); - void completePath(size_t, std::string_view prefix); void completeDir(size_t, std::string_view prefix); diff -Nru nix-2.8.0/src/libutil/callback.hh nix-2.16.1+dfsg/src/libutil/callback.hh --- nix-2.8.0/src/libutil/callback.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/callback.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,13 +1,16 @@ #pragma once +///@file #include #include namespace nix { -/* A callback is a wrapper around a lambda that accepts a valid of - type T or an exception. (We abuse std::future to pass the value or - exception.) */ +/** + * A callback is a wrapper around a lambda that accepts a valid of + * type T or an exception. (We abuse std::future to pass the value or + * exception.) + */ template class Callback { diff -Nru nix-2.8.0/src/libutil/canon-path.cc nix-2.16.1+dfsg/src/libutil/canon-path.cc --- nix-2.8.0/src/libutil/canon-path.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/canon-path.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,134 @@ +#include "canon-path.hh" +#include "util.hh" + +namespace nix { + +CanonPath CanonPath::root = CanonPath("/"); + +CanonPath::CanonPath(std::string_view raw) + : path(absPath((Path) raw, "/")) +{ } + +CanonPath::CanonPath(std::string_view raw, const CanonPath & root) + : path(absPath((Path) raw, root.abs())) +{ } + +CanonPath CanonPath::fromCwd(std::string_view path) +{ + return CanonPath(unchecked_t(), absPath((Path) path)); +} + +std::optional CanonPath::parent() const +{ + if (isRoot()) return std::nullopt; + return CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, path.rfind('/')))); +} + +void CanonPath::pop() +{ + assert(!isRoot()); + path.resize(std::max((size_t) 1, path.rfind('/'))); +} + +bool CanonPath::isWithin(const CanonPath & parent) const +{ + return !( + path.size() < parent.path.size() + || path.substr(0, parent.path.size()) != parent.path + || (parent.path.size() > 1 && path.size() > parent.path.size() + && path[parent.path.size()] != '/')); +} + +CanonPath CanonPath::removePrefix(const CanonPath & prefix) const +{ + assert(isWithin(prefix)); + if (prefix.isRoot()) return *this; + if (path.size() == prefix.path.size()) return root; + return CanonPath(unchecked_t(), path.substr(prefix.path.size())); +} + +void CanonPath::extend(const CanonPath & x) +{ + if (x.isRoot()) return; + if (isRoot()) + path += x.rel(); + else + path += x.abs(); +} + +CanonPath CanonPath::operator + (const CanonPath & x) const +{ + auto res = *this; + res.extend(x); + return res; +} + +void CanonPath::push(std::string_view c) +{ + assert(c.find('/') == c.npos); + assert(c != "." && c != ".."); + if (!isRoot()) path += '/'; + path += c; +} + +CanonPath CanonPath::operator + (std::string_view c) const +{ + auto res = *this; + res.push(c); + return res; +} + +bool CanonPath::isAllowed(const std::set & allowed) const +{ + /* Check if `this` is an exact match or the parent of an + allowed path. */ + auto lb = allowed.lower_bound(*this); + if (lb != allowed.end()) { + if (lb->isWithin(*this)) + return true; + } + + /* Check if a parent of `this` is allowed. */ + auto path = *this; + while (!path.isRoot()) { + path.pop(); + if (allowed.count(path)) + return true; + } + + return false; +} + +std::ostream & operator << (std::ostream & stream, const CanonPath & path) +{ + stream << path.abs(); + return stream; +} + +std::string CanonPath::makeRelative(const CanonPath & path) const +{ + auto p1 = begin(); + auto p2 = path.begin(); + + for (; p1 != end() && p2 != path.end() && *p1 == *p2; ++p1, ++p2) ; + + if (p1 == end() && p2 == path.end()) + return "."; + else if (p1 == end()) + return std::string(p2.remaining); + else { + std::string res; + while (p1 != end()) { + ++p1; + if (!res.empty()) res += '/'; + res += ".."; + } + if (p2 != path.end()) { + if (!res.empty()) res += '/'; + res += p2.remaining; + } + return res; + } +} + +} diff -Nru nix-2.8.0/src/libutil/canon-path.hh nix-2.16.1+dfsg/src/libutil/canon-path.hh --- nix-2.8.0/src/libutil/canon-path.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/canon-path.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,206 @@ +#pragma once +///@file + +#include +#include +#include +#include +#include + +namespace nix { + +/** + * A canonical representation of a path. It ensures the following: + * + * - It always starts with a slash. + * + * - It never ends with a slash, except if the path is "/". + * + * - A slash is never followed by a slash (i.e. no empty components). + * + * - There are no components equal to '.' or '..'. + * + * Note that the path does not need to correspond to an actually + * existing path, and there is no guarantee that symlinks are + * resolved. + */ +class CanonPath +{ + std::string path; + +public: + + /** + * Construct a canon path from a non-canonical path. Any '.', '..' + * or empty components are removed. + */ + CanonPath(std::string_view raw); + + explicit CanonPath(const char * raw) + : CanonPath(std::string_view(raw)) + { } + + struct unchecked_t { }; + + CanonPath(unchecked_t _, std::string path) + : path(std::move(path)) + { } + + static CanonPath fromCwd(std::string_view path = "."); + + static CanonPath root; + + /** + * If `raw` starts with a slash, return + * `CanonPath(raw)`. Otherwise return a `CanonPath` representing + * `root + "/" + raw`. + */ + CanonPath(std::string_view raw, const CanonPath & root); + + bool isRoot() const + { return path.size() <= 1; } + + explicit operator std::string_view() const + { return path; } + + const std::string & abs() const + { return path; } + + /** + * Like abs(), but return an empty string if this path is + * '/'. Thus the returned string never ends in a slash. + */ + const std::string & absOrEmpty() const + { + const static std::string epsilon; + return isRoot() ? epsilon : path; + } + + const char * c_str() const + { return path.c_str(); } + + std::string_view rel() const + { return ((std::string_view) path).substr(1); } + + struct Iterator + { + std::string_view remaining; + size_t slash; + + Iterator(std::string_view remaining) + : remaining(remaining) + , slash(remaining.find('/')) + { } + + bool operator != (const Iterator & x) const + { return remaining.data() != x.remaining.data(); } + + bool operator == (const Iterator & x) const + { return !(*this != x); } + + const std::string_view operator * () const + { return remaining.substr(0, slash); } + + void operator ++ () + { + if (slash == remaining.npos) + remaining = remaining.substr(remaining.size()); + else { + remaining = remaining.substr(slash + 1); + slash = remaining.find('/'); + } + } + }; + + Iterator begin() const { return Iterator(rel()); } + Iterator end() const { return Iterator(rel().substr(path.size() - 1)); } + + std::optional parent() const; + + /** + * Remove the last component. Panics if this path is the root. + */ + void pop(); + + std::optional dirOf() const + { + if (isRoot()) return std::nullopt; + return ((std::string_view) path).substr(0, path.rfind('/')); + } + + std::optional baseName() const + { + if (isRoot()) return std::nullopt; + return ((std::string_view) path).substr(path.rfind('/') + 1); + } + + bool operator == (const CanonPath & x) const + { return path == x.path; } + + bool operator != (const CanonPath & x) const + { return path != x.path; } + + /** + * Compare paths lexicographically except that path separators + * are sorted before any other character. That is, in the sorted order + * a directory is always followed directly by its children. For + * instance, 'foo' < 'foo/bar' < 'foo!'. + */ + bool operator < (const CanonPath & x) const + { + auto i = path.begin(); + auto j = x.path.begin(); + for ( ; i != path.end() && j != x.path.end(); ++i, ++j) { + auto c_i = *i; + if (c_i == '/') c_i = 0; + auto c_j = *j; + if (c_j == '/') c_j = 0; + if (c_i < c_j) return true; + if (c_i > c_j) return false; + } + return i == path.end() && j != x.path.end(); + } + + /** + * Return true if `this` is equal to `parent` or a child of + * `parent`. + */ + bool isWithin(const CanonPath & parent) const; + + CanonPath removePrefix(const CanonPath & prefix) const; + + /** + * Append another path to this one. + */ + void extend(const CanonPath & x); + + /** + * Concatenate two paths. + */ + CanonPath operator + (const CanonPath & x) const; + + /** + * Add a path component to this one. It must not contain any slashes. + */ + void push(std::string_view c); + + CanonPath operator + (std::string_view c) const; + + /** + * Check whether access to this path is allowed, which is the case + * if 1) `this` is within any of the `allowed` paths; or 2) any of + * the `allowed` paths are within `this`. (The latter condition + * ensures access to the parents of allowed paths.) + */ + bool isAllowed(const std::set & allowed) const; + + /** + * Return a representation `x` of `path` relative to `this`, i.e. + * `CanonPath(this.makeRelative(x), this) == path`. + */ + std::string makeRelative(const CanonPath & path) const; +}; + +std::ostream & operator << (std::ostream & stream, const CanonPath & path); + +} diff -Nru nix-2.8.0/src/libutil/cgroup.cc nix-2.16.1+dfsg/src/libutil/cgroup.cc --- nix-2.8.0/src/libutil/cgroup.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/cgroup.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,148 @@ +#if __linux__ + +#include "cgroup.hh" +#include "util.hh" +#include "finally.hh" + +#include +#include +#include +#include +#include + +#include +#include + +namespace nix { + +std::optional getCgroupFS() +{ + static auto res = [&]() -> std::optional { + auto fp = fopen("/proc/mounts", "r"); + if (!fp) return std::nullopt; + Finally delFP = [&]() { fclose(fp); }; + while (auto ent = getmntent(fp)) + if (std::string_view(ent->mnt_type) == "cgroup2") + return ent->mnt_dir; + + return std::nullopt; + }(); + return res; +} + +// FIXME: obsolete, check for cgroup2 +std::map getCgroups(const Path & cgroupFile) +{ + std::map cgroups; + + for (auto & line : tokenizeString>(readFile(cgroupFile), "\n")) { + static std::regex regex("([0-9]+):([^:]*):(.*)"); + std::smatch match; + if (!std::regex_match(line, match, regex)) + throw Error("invalid line '%s' in '%s'", line, cgroupFile); + + std::string name = hasPrefix(std::string(match[2]), "name=") ? std::string(match[2], 5) : match[2]; + cgroups.insert_or_assign(name, match[3]); + } + + return cgroups; +} + +static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats) +{ + if (!pathExists(cgroup)) return {}; + + auto procsFile = cgroup + "/cgroup.procs"; + + if (!pathExists(procsFile)) + throw Error("'%s' is not a cgroup", cgroup); + + /* Use the fast way to kill every process in a cgroup, if + available. */ + auto killFile = cgroup + "/cgroup.kill"; + if (pathExists(killFile)) + writeFile(killFile, "1"); + + /* Otherwise, manually kill every process in the subcgroups and + this cgroup. */ + for (auto & entry : readDirectory(cgroup)) { + if (entry.type != DT_DIR) continue; + destroyCgroup(cgroup + "/" + entry.name, false); + } + + int round = 1; + + std::unordered_set pidsShown; + + while (true) { + auto pids = tokenizeString>(readFile(procsFile)); + + if (pids.empty()) break; + + if (round > 20) + throw Error("cannot kill cgroup '%s'", cgroup); + + for (auto & pid_s : pids) { + pid_t pid; + if (auto o = string2Int(pid_s)) + pid = *o; + else + throw Error("invalid pid '%s'", pid); + if (pidsShown.insert(pid).second) { + try { + auto cmdline = readFile(fmt("/proc/%d/cmdline", pid)); + using namespace std::string_literals; + warn("killing stray builder process %d (%s)...", + pid, trim(replaceStrings(cmdline, "\0"s, " "))); + } catch (SysError &) { + } + } + // FIXME: pid wraparound + if (kill(pid, SIGKILL) == -1 && errno != ESRCH) + throw SysError("killing member %d of cgroup '%s'", pid, cgroup); + } + + auto sleep = std::chrono::milliseconds((int) std::pow(2.0, std::min(round, 10))); + if (sleep.count() > 100) + printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup); + std::this_thread::sleep_for(sleep); + round++; + } + + CgroupStats stats; + + if (returnStats) { + auto cpustatPath = cgroup + "/cpu.stat"; + + if (pathExists(cpustatPath)) { + for (auto & line : tokenizeString>(readFile(cpustatPath), "\n")) { + std::string_view userPrefix = "user_usec "; + if (hasPrefix(line, userPrefix)) { + auto n = string2Int(line.substr(userPrefix.size())); + if (n) stats.cpuUser = std::chrono::microseconds(*n); + } + + std::string_view systemPrefix = "system_usec "; + if (hasPrefix(line, systemPrefix)) { + auto n = string2Int(line.substr(systemPrefix.size())); + if (n) stats.cpuSystem = std::chrono::microseconds(*n); + } + } + } + + } + + if (rmdir(cgroup.c_str()) == -1) + throw SysError("deleting cgroup '%s'", cgroup); + + return stats; +} + +CgroupStats destroyCgroup(const Path & cgroup) +{ + return destroyCgroup(cgroup, true); +} + +} + +#endif diff -Nru nix-2.8.0/src/libutil/cgroup.hh nix-2.16.1+dfsg/src/libutil/cgroup.hh --- nix-2.8.0/src/libutil/cgroup.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/cgroup.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,32 @@ +#pragma once +///@file + +#if __linux__ + +#include +#include + +#include "types.hh" + +namespace nix { + +std::optional getCgroupFS(); + +std::map getCgroups(const Path & cgroupFile); + +struct CgroupStats +{ + std::optional cpuUser, cpuSystem; +}; + +/** + * Destroy the cgroup denoted by 'path'. The postcondition is that + * 'path' does not exist, and thus any processes in the cgroup have + * been killed. Also return statistics from the cgroup just before + * destruction. + */ +CgroupStats destroyCgroup(const Path & cgroup); + +} + +#endif diff -Nru nix-2.8.0/src/libutil/chunked-vector.hh nix-2.16.1+dfsg/src/libutil/chunked-vector.hh --- nix-2.8.0/src/libutil/chunked-vector.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/chunked-vector.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,73 @@ +#pragma once +///@file + +#include +#include +#include +#include + +namespace nix { + +/** + * Provides an indexable container like vector<> with memory overhead + * guarantees like list<> by allocating storage in chunks of ChunkSize + * elements instead of using a contiguous memory allocation like vector<> + * does. Not using a single vector that is resized reduces memory overhead + * on large data sets by on average (growth factor)/2, mostly + * eliminates copies within the vector during resizing, and provides stable + * references to its elements. + */ +template +class ChunkedVector { +private: + uint32_t size_ = 0; + std::vector> chunks; + + /** + * Keep this out of the ::add hot path + */ + [[gnu::noinline]] + auto & addChunk() + { + if (size_ >= std::numeric_limits::max() - ChunkSize) + abort(); + chunks.emplace_back(); + chunks.back().reserve(ChunkSize); + return chunks.back(); + } + +public: + ChunkedVector(uint32_t reserve) + { + chunks.reserve(reserve); + addChunk(); + } + + uint32_t size() const { return size_; } + + std::pair add(T value) + { + const auto idx = size_++; + auto & chunk = [&] () -> auto & { + if (auto & back = chunks.back(); back.size() < ChunkSize) + return back; + return addChunk(); + }(); + auto & result = chunk.emplace_back(std::move(value)); + return {result, idx}; + } + + const T & operator[](uint32_t idx) const + { + return chunks[idx / ChunkSize][idx % ChunkSize]; + } + + template + void forEach(Fn fn) const + { + for (const auto & c : chunks) + for (const auto & e : c) + fn(e); + } +}; +} diff -Nru nix-2.8.0/src/libutil/closure.hh nix-2.16.1+dfsg/src/libutil/closure.hh --- nix-2.8.0/src/libutil/closure.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/closure.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,3 +1,6 @@ +#pragma once +///@file + #include #include #include "sync.hh" diff -Nru nix-2.8.0/src/libutil/comparator.hh nix-2.16.1+dfsg/src/libutil/comparator.hh --- nix-2.8.0/src/libutil/comparator.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/comparator.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,8 @@ #pragma once +///@file -/* Awfull hacky generation of the comparison operators by doing a lexicographic +/** + * Awful hacky generation of the comparison operators by doing a lexicographic * comparison between the choosen fields. * * ``` @@ -15,12 +17,12 @@ * } * ``` */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ +#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, ...) \ bool operator COMPARATOR(const MY_TYPE& other) const { \ - const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple( FIELDS ); \ - me = &other; \ - auto fields2 = std::make_tuple( FIELDS ); \ + __VA_OPT__(const MY_TYPE* me = this;) \ + auto fields1 = std::make_tuple( __VA_ARGS__ ); \ + __VA_OPT__(me = &other;) \ + auto fields2 = std::make_tuple( __VA_ARGS__ ); \ return fields1 COMPARATOR fields2; \ } #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) diff -Nru nix-2.8.0/src/libutil/compression.cc nix-2.16.1+dfsg/src/libutil/compression.cc --- nix-2.8.0/src/libutil/compression.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/compression.cc 2023-06-06 10:55:23.000000000 +0000 @@ -23,7 +23,7 @@ { uint8_t outbuf[32 * 1024]; - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { const size_t CHUNK_SIZE = sizeof(outbuf) << 2; while (!data.empty()) { @@ -103,7 +103,7 @@ throw Error(reason, archive_error_string(this->archive)); } - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { ssize_t result = archive_write_data(archive, data.data(), data.length()); if (result <= 0) check(result); @@ -136,7 +136,7 @@ warn("requested compression level '%d' not supported by compression method 'none'", level); } void finish() override { flush(); } - void write(std::string_view data) override { nextSink(data); } + void writeUnbuffered(std::string_view data) override { nextSink(data); } }; struct BrotliDecompressionSink : ChunkedCompressionSink diff -Nru nix-2.8.0/src/libutil/compression.hh nix-2.16.1+dfsg/src/libutil/compression.hh --- nix-2.8.0/src/libutil/compression.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/compression.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" #include "types.hh" @@ -11,7 +12,7 @@ struct CompressionSink : BufferedSink, FinishSink { using BufferedSink::operator (); - using BufferedSink::write; + using BufferedSink::writeUnbuffered; using FinishSink::finish; }; diff -Nru nix-2.8.0/src/libutil/compute-levels.hh nix-2.16.1+dfsg/src/libutil/compute-levels.hh --- nix-2.8.0/src/libutil/compute-levels.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/compute-levels.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "types.hh" namespace nix { diff -Nru nix-2.8.0/src/libutil/config.cc nix-2.16.1+dfsg/src/libutil/config.cc --- nix-2.8.0/src/libutil/config.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/config.cc 2023-06-06 10:55:23.000000000 +0000 @@ -3,6 +3,8 @@ #include "abstract-setting-to-json.hh" #include "experimental-features.hh" +#include "config-impl.hh" + #include namespace nix { @@ -80,6 +82,8 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { unsigned int pos = 0; + std::vector> parsedContents; + while (pos < contents.size()) { std::string line; while (pos < contents.size() && contents[pos] != '\n') @@ -125,8 +129,21 @@ auto i = tokens.begin(); advance(i, 2); - set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + parsedContents.push_back({ + name, + concatStringsSep(" ", Strings(i, tokens.end())), + }); }; + + // First apply experimental-feature related settings + for (auto & [name, value] : parsedContents) + if (name == "experimental-features" || name == "extra-experimental-features") + set(name, value); + + // Then apply other settings + for (auto & [name, value] : parsedContents) + if (name != "experimental-features" && name != "extra-experimental-features") + set(name, value); } void AbstractConfig::applyConfigFile(const Path & path) @@ -147,9 +164,8 @@ { auto res = nlohmann::json::object(); for (auto & s : _settings) - if (!s.second.isAlias) { + if (!s.second.isAlias) res.emplace(s.first, s.second.setting->toJSON()); - } return res; } @@ -157,24 +173,28 @@ { auto res = std::string(); for (auto & s : _settings) - if (!s.second.isAlias) { + if (s.second.isAlias) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); - } return res; } void Config::convertToArgs(Args & args, const std::string & category) { - for (auto & s : _settings) + for (auto & s : _settings) { if (!s.second.isAlias) s.second.setting->convertToArg(args, category); + } } AbstractSetting::AbstractSetting( const std::string & name, const std::string & description, - const std::set & aliases) - : name(name), description(stripIndentation(description)), aliases(aliases) + const std::set & aliases, + std::optional experimentalFeature) + : name(name) + , description(stripIndentation(description)) + , aliases(aliases) + , experimentalFeature(experimentalFeature) { } @@ -188,6 +208,10 @@ std::map obj; obj.emplace("description", description); obj.emplace("aliases", aliases); + if (experimentalFeature) + obj.emplace("experimentalFeature", *experimentalFeature); + else + obj.emplace("experimentalFeature", nullptr); return obj; } @@ -196,12 +220,6 @@ } template -bool BaseSetting::isAppendable() -{ - return false; -} - -template void BaseSetting::convertToArg(Args & args, const std::string & category) { args.addFlag({ @@ -209,7 +227,8 @@ .description = fmt("Set the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overridden = true; set(s); }}, + .handler = {[this](std::string s) { overridden = true; set(s); }}, + .experimentalFeature = experimentalFeature, }); if (isAppendable()) @@ -218,13 +237,14 @@ .description = fmt("Append to the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overridden = true; set(s, true); }}, + .handler = {[this](std::string s) { overridden = true; set(s, true); }}, + .experimentalFeature = experimentalFeature, }); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> std::string BaseSetting::parse(const std::string & str) const { - value = str; + return str; } template<> std::string BaseSetting::to_string() const @@ -233,11 +253,11 @@ } template -void BaseSetting::set(const std::string & str, bool append) +T BaseSetting::parse(const std::string & str) const { static_assert(std::is_integral::value, "Integer required."); if (auto n = string2Int(str)) - value = *n; + return *n; else throw UsageError("setting '%s' has invalid value '%s'", name, str); } @@ -249,12 +269,12 @@ return std::to_string(value); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> bool BaseSetting::parse(const std::string & str) const { if (str == "true" || str == "yes" || str == "1") - value = true; + return true; else if (str == "false" || str == "no" || str == "0") - value = false; + return false; else throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); } @@ -270,26 +290,27 @@ .longName = name, .description = fmt("Enable the `%s` setting.", name), .category = category, - .handler = {[=]() { override(true); }} + .handler = {[this]() { override(true); }}, + .experimentalFeature = experimentalFeature, }); args.addFlag({ .longName = "no-" + name, .description = fmt("Disable the `%s` setting.", name), .category = category, - .handler = {[=]() { override(false); }} + .handler = {[this]() { override(false); }}, + .experimentalFeature = experimentalFeature, }); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> Strings BaseSetting::parse(const std::string & str) const { - auto ss = tokenizeString(str); - if (!append) value.clear(); - for (auto & s : ss) value.push_back(std::move(s)); + return tokenizeString(str); } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(Strings && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && s : std::move(newValue)) value.push_back(std::move(s)); } template<> std::string BaseSetting::to_string() const @@ -297,16 +318,16 @@ return concatStringsSep(" ", value); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> StringSet BaseSetting::parse(const std::string & str) const { - if (!append) value.clear(); - for (auto & s : tokenizeString(str)) - value.insert(s); + return tokenizeString(str); } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && s : std::move(newValue)) + value.insert(s); } template<> std::string BaseSetting::to_string() const @@ -314,21 +335,24 @@ return concatStringsSep(" ", value); } -template<> void BaseSetting>::set(const std::string & str, bool append) +template<> std::set BaseSetting>::parse(const std::string & str) const { - if (!append) value.clear(); + std::set res; for (auto & s : tokenizeString(str)) { auto thisXpFeature = parseExperimentalFeature(s); if (thisXpFeature) - value.insert(thisXpFeature.value()); + res.insert(thisXpFeature.value()); else warn("unknown experimental feature '%s'", s); } + return res; } -template<> bool BaseSetting>::isAppendable() +template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && s : std::move(newValue)) + value.insert(s); } template<> std::string BaseSetting>::to_string() const @@ -339,20 +363,23 @@ return concatStringsSep(" ", stringifiedXpFeatures); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> StringMap BaseSetting::parse(const std::string & str) const { - if (!append) value.clear(); + StringMap res; for (auto & s : tokenizeString(str)) { auto eq = s.find_first_of('='); if (std::string::npos != eq) - value.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); + res.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); // else ignored } + return res; } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && [k, v] : std::move(newValue)) + value.emplace(std::move(k), std::move(v)); } template<> std::string BaseSetting::to_string() const @@ -376,15 +403,15 @@ template class BaseSetting; template class BaseSetting>; -void PathSetting::set(const std::string & str, bool append) +Path PathSetting::parse(const std::string & str) const { if (str == "") { if (allowEmpty) - value = ""; + return ""; else throw UsageError("setting '%s' cannot be empty", name); } else - value = canonPath(str); + return canonPath(str); } bool GlobalConfig::set(const std::string & name, const std::string & value) @@ -444,4 +471,30 @@ configRegistrations->emplace_back(config); } +ExperimentalFeatureSettings experimentalFeatureSettings; + +static GlobalConfig::Register rSettings(&experimentalFeatureSettings); + +bool ExperimentalFeatureSettings::isEnabled(const ExperimentalFeature & feature) const +{ + auto & f = experimentalFeatures.get(); + return std::find(f.begin(), f.end(), feature) != f.end(); +} + +void ExperimentalFeatureSettings::require(const ExperimentalFeature & feature) const +{ + if (!isEnabled(feature)) + throw MissingExperimentalFeature(feature); +} + +bool ExperimentalFeatureSettings::isEnabled(const std::optional & feature) const +{ + return !feature || isEnabled(*feature); +} + +void ExperimentalFeatureSettings::require(const std::optional & feature) const +{ + if (feature) require(*feature); +} + } diff -Nru nix-2.8.0/src/libutil/config.hh nix-2.16.1+dfsg/src/libutil/config.hh --- nix-2.8.0/src/libutil/config.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/config.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,12 +1,14 @@ +#pragma once +///@file + #include #include #include -#include "types.hh" - #include -#pragma once +#include "types.hh" +#include "experimental-features.hh" namespace nix { @@ -123,21 +125,21 @@ void reapplyUnknownSettings(); }; -/* A class to simplify providing configuration settings. The typical - use is to inherit Config and add Setting members: - - class MyClass : private Config - { - Setting foo{this, 123, "foo", "the number of foos to use"}; - Setting bar{this, "blabla", "bar", "the name of the bar"}; - - MyClass() : Config(readConfigFile("/etc/my-app.conf")) - { - std::cout << foo << "\n"; // will print 123 unless overridden - } - }; -*/ - +/** + * A class to simplify providing configuration settings. The typical + * use is to inherit Config and add Setting members: + * + * class MyClass : private Config + * { + * Setting foo{this, 123, "foo", "the number of foos to use"}; + * Setting bar{this, "blabla", "bar", "the name of the bar"}; + * + * MyClass() : Config(readConfigFile("/etc/my-app.conf")) + * { + * std::cout << foo << "\n"; // will print 123 unless overridden + * } + * }; + */ class Config : public AbstractConfig { friend class AbstractSetting; @@ -194,12 +196,15 @@ bool overridden = false; + std::optional experimentalFeature; + protected: AbstractSetting( const std::string & name, const std::string & description, - const std::set & aliases); + const std::set & aliases, + std::optional experimentalFeature = std::nullopt); virtual ~AbstractSetting() { @@ -210,8 +215,11 @@ virtual void set(const std::string & value, bool append = false) = 0; - virtual bool isAppendable() - { return false; } + /** + * Whether the type is appendable; i.e. whether the `append` + * parameter to `set()` is allowed to be `true`. + */ + virtual bool isAppendable() = 0; virtual std::string to_string() const = 0; @@ -224,7 +232,9 @@ bool isOverridden() const { return overridden; } }; -/* A setting of type T. */ +/** + * A setting of type T. + */ template class BaseSetting : public AbstractSetting { @@ -234,14 +244,32 @@ const T defaultValue; const bool documentDefault; + /** + * Parse the string into a `T`. + * + * Used by `set()`. + */ + virtual T parse(const std::string & str) const; + + /** + * Append or overwrite `value` with `newValue`. + * + * Some types to do not support appending in which case `append` + * should never be passed. The default handles this case. + * + * @param append Whether to append or overwrite. + */ + virtual void appendOrSet(T && newValue, bool append); + public: BaseSetting(const T & def, const bool documentDefault, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : AbstractSetting(name, description, aliases) + const std::set & aliases = {}, + std::optional experimentalFeature = std::nullopt) + : AbstractSetting(name, description, aliases, experimentalFeature) , value(def) , defaultValue(def) , documentDefault(documentDefault) @@ -250,15 +278,35 @@ operator const T &() const { return value; } operator T &() { return value; } const T & get() const { return value; } - bool operator ==(const T & v2) const { return value == v2; } - bool operator !=(const T & v2) const { return value != v2; } - void operator =(const T & v) { assign(v); } + template + bool operator ==(const U & v2) const { return value == v2; } + template + bool operator !=(const U & v2) const { return value != v2; } + template + void operator =(const U & v) { assign(v); } virtual void assign(const T & v) { value = v; } - void setDefault(const T & v) { if (!overridden) value = v; } + template + void setDefault(const U & v) { if (!overridden) value = v; } + + /** + * Require any experimental feature the setting depends on + * + * Uses `parse()` to get the value from `str`, and `appendOrSet()` + * to set it. + */ + void set(const std::string & str, bool append = false) override final; - void set(const std::string & str, bool append = false) override; + /** + * C++ trick; This is template-specialized to compile-time indicate whether + * the type is appendable. + */ + struct trait; - bool isAppendable() override; + /** + * Always defined based on the C++ magic + * with `trait` above. + */ + bool isAppendable() override final; virtual void override(const T & v) { @@ -292,8 +340,9 @@ const std::string & name, const std::string & description, const std::set & aliases = {}, - const bool documentDefault = true) - : BaseSetting(def, documentDefault, name, description, aliases) + const bool documentDefault = true, + std::optional experimentalFeature = std::nullopt) + : BaseSetting(def, documentDefault, name, description, aliases, experimentalFeature) { options->addSetting(this); } @@ -301,8 +350,10 @@ void operator =(const T & v) { this->assign(v); } }; -/* A special setting for Paths. These are automatically canonicalised - (e.g. "/foo//bar/" becomes "/foo/bar"). */ +/** + * A special setting for Paths. These are automatically canonicalised + * (e.g. "/foo//bar/" becomes "/foo/bar"). + */ class PathSetting : public BaseSetting { bool allowEmpty; @@ -321,7 +372,7 @@ options->addSetting(this); } - void set(const std::string & str, bool append = false) override; + Path parse(const std::string & str) const override; Path operator +(const char * p) const { return value + p; } @@ -353,4 +404,52 @@ extern GlobalConfig globalConfig; + +struct ExperimentalFeatureSettings : Config { + + Setting> experimentalFeatures{ + this, {}, "experimental-features", + R"( + Experimental features that are enabled. + + Example: + + ``` + experimental-features = nix-command flakes + ``` + + The following experimental features are available: + + {{#include experimental-features-shortlist.md}} + + Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md). + )"}; + + /** + * Check whether the given experimental feature is enabled. + */ + bool isEnabled(const ExperimentalFeature &) const; + + /** + * Require an experimental feature be enabled, throwing an error if it is + * not. + */ + void require(const ExperimentalFeature &) const; + + /** + * `std::nullopt` pointer means no feature, which means there is nothing that could be + * disabled, and so the function returns true in that case. + */ + bool isEnabled(const std::optional &) const; + + /** + * `std::nullopt` pointer means no feature, which means there is nothing that could be + * disabled, and so the function does nothing in that case. + */ + void require(const std::optional &) const; +}; + +// FIXME: don't use a global variable. +extern ExperimentalFeatureSettings experimentalFeatureSettings; + } diff -Nru nix-2.8.0/src/libutil/config-impl.hh nix-2.16.1+dfsg/src/libutil/config-impl.hh --- nix-2.8.0/src/libutil/config-impl.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/config-impl.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,71 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * One only needs to include this when one is declaring a + * `BaseClass` setting, or as derived class of such an + * instantiation. + */ + +#include "config.hh" + +namespace nix { + +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting>::trait +{ + static constexpr bool appendable = true; +}; + +template +struct BaseSetting::trait +{ + static constexpr bool appendable = false; +}; + +template +bool BaseSetting::isAppendable() +{ + return trait::appendable; +} + +template<> void BaseSetting::appendOrSet(Strings && newValue, bool append); +template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append); +template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append); +template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append); + +template +void BaseSetting::appendOrSet(T && newValue, bool append) +{ + static_assert(!trait::appendable, "using default `appendOrSet` implementation with an appendable type"); + assert(!append); + value = std::move(newValue); +} + +template +void BaseSetting::set(const std::string & str, bool append) +{ + if (experimentalFeatureSettings.isEnabled(experimentalFeature)) + appendOrSet(parse(str), append); + else { + assert(experimentalFeature); + warn("Ignoring setting '%s' because experimental feature '%s' is not enabled", + name, + showExperimentalFeature(*experimentalFeature)); + } +} + +} diff -Nru nix-2.8.0/src/libutil/error.cc nix-2.16.1+dfsg/src/libutil/error.cc --- nix-2.8.0/src/libutil/error.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/error.cc 2023-06-06 10:55:23.000000000 +0000 @@ -9,9 +9,9 @@ const std::string nativeSystem = SYSTEM; -void BaseError::addTrace(std::optional e, hintformat hint) +void BaseError::addTrace(std::shared_ptr && e, hintformat hint, bool frame) { - err.traces.push_front(Trace { .pos = e, .hint = hint }); + err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame }); } // c++ std::exception descendants must have a 'const char* what()' function. @@ -30,91 +30,46 @@ std::optional ErrorInfo::programName = std::nullopt; -std::ostream & operator<<(std::ostream & os, const hintformat & hf) +std::ostream & operator <<(std::ostream & os, const hintformat & hf) { return os << hf.str(); } -std::string showErrPos(const ErrPos & errPos) +std::ostream & operator <<(std::ostream & str, const AbstractPos & pos) { - if (errPos.line > 0) { - if (errPos.column > 0) { - return fmt("%d:%d", errPos.line, errPos.column); - } else { - return fmt("%d", errPos.line); - } - } - else { - return ""; - } + pos.print(str); + str << ":" << pos.line; + if (pos.column > 0) + str << ":" << pos.column; + return str; } -std::optional getCodeLines(const ErrPos & errPos) +std::optional AbstractPos::getCodeLines() const { - if (errPos.line <= 0) + if (line == 0) return std::nullopt; - if (errPos.origin == foFile) { - LinesOfCode loc; - try { - // FIXME: when running as the daemon, make sure we don't - // open a file to which the client doesn't have access. - AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) return {}; - - // count the newlines. - int count = 0; - std::string line; - int pl = errPos.line - 1; - do - { - line = readLine(fd.get()); - ++count; - if (count < pl) - ; - else if (count == pl) - loc.prevLineOfCode = line; - else if (count == pl + 1) - loc.errLineOfCode = line; - else if (count == pl + 2) { - loc.nextLineOfCode = line; - break; - } - } while (true); - return loc; - } - catch (EndOfFile & eof) { - if (loc.errLineOfCode.has_value()) - return loc; - else - return std::nullopt; - } - catch (std::exception & e) { - return std::nullopt; - } - } else { - std::istringstream iss(errPos.file); + if (auto source = getSource()) { + + std::istringstream iss(*source); // count the newlines. int count = 0; - std::string line; - int pl = errPos.line - 1; + std::string curLine; + int pl = line - 1; LinesOfCode loc; - do - { - std::getline(iss, line); + do { + std::getline(iss, curLine); ++count; if (count < pl) - { ; - } else if (count == pl) { - loc.prevLineOfCode = line; + loc.prevLineOfCode = curLine; } else if (count == pl + 1) { - loc.errLineOfCode = line; + loc.errLineOfCode = curLine; } else if (count == pl + 2) { - loc.nextLineOfCode = line; + loc.nextLineOfCode = curLine; break; } @@ -124,12 +79,14 @@ return loc; } + + return std::nullopt; } // print lines of code to the ostream, indicating the error column. void printCodeLines(std::ostream & out, const std::string & prefix, - const ErrPos & errPos, + const AbstractPos & errPos, const LinesOfCode & loc) { // previous line of code. @@ -176,28 +133,6 @@ } } -void printAtPos(const ErrPos & pos, std::ostream & out) -{ - if (pos) { - switch (pos.origin) { - case foFile: { - out << fmt(ANSI_BLUE "at " ANSI_WARNING "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos)); - break; - } - case foString: { - out << fmt(ANSI_BLUE "at " ANSI_WARNING "«string»:%s" ANSI_NORMAL ":", showErrPos(pos)); - break; - } - case foStdin: { - out << fmt(ANSI_BLUE "at " ANSI_WARNING "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos)); - break; - } - default: - throw Error("invalid FileOrigin in errPos"); - } - } -} - static std::string indent(std::string_view indentFirst, std::string_view indentRest, std::string_view s) { std::string res; @@ -262,49 +197,160 @@ prefix += ":" ANSI_NORMAL " "; std::ostringstream oss; - oss << einfo.msg << "\n"; - if (einfo.errPos.has_value() && *einfo.errPos) { - oss << "\n"; - printAtPos(*einfo.errPos, oss); + auto noSource = ANSI_ITALIC " (source not available)" ANSI_NORMAL "\n"; - auto loc = getCodeLines(*einfo.errPos); + /* + * Traces + * ------ + * + * The semantics of traces is a bit weird. We have only one option to + * print them and to make them verbose (--show-trace). In the code they + * are always collected, but they are not printed by default. The code + * also collects more traces when the option is on. This means that there + * is no way to print the simplified traces at all. + * + * I (layus) designed the code to attach positions to a restricted set of + * messages. This means that we have a lot of traces with no position at + * all, including most of the base error messages. For example "type + * error: found a string while a set was expected" has no position, but + * will come with several traces detailing it's precise relation to the + * closest know position. This makes erroring without printing traces + * quite useless. + * + * This is why I introduced the idea to always print a few traces on + * error. The number 3 is quite arbitrary, and was selected so as not to + * clutter the console on error. For the same reason, a trace with an + * error position takes more space, and counts as two traces towards the + * limit. + * + * The rest is truncated, unless --show-trace is passed. This preserves + * the same bad semantics of --show-trace to both show the trace and + * augment it with new data. Not too sure what is the best course of + * action. + * + * The issue is that it is fundamentally hard to provide a trace for a + * lazy language. The trace will only cover the current spine of the + * evaluation, missing things that have been evaluated before. For + * example, most type errors are hard to inspect because there is not + * trace for the faulty value. These errors should really print the faulty + * value itself. + * + * In function calls, the --show-trace flag triggers extra traces for each + * function invocation. These work as scopes, allowing to follow the + * current spine of the evaluation graph. Without that flag, the error + * trace should restrict itself to a restricted prefix of that trace, + * until the first scope. If we ever get to such a precise error + * reporting, there would be no need to add an arbitrary limit here. We + * could always print the full trace, and it would just be small without + * the flag. + * + * One idea I had is for XxxError.addTrace() to perform nothing if one + * scope has already been traced. Alternatively, we could stop here when + * we encounter such a scope instead of after an arbitrary number of + * traces. This however requires to augment traces with the notion of + * "scope". + * + * This is particularly visible in code like evalAttrs(...) where we have + * to make a decision between the two following options. + * + * ``` long traces + * inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, std::string_view errorCtx) + * { + * try { + * e->eval(*this, env, v); + * if (v.type() != nAttrs) + * throwTypeError("value is %1% while a set was expected", v); + * } catch (Error & e) { + * e.addTrace(pos, errorCtx); + * throw; + * } + * } + * ``` + * + * ``` short traces + * inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, std::string_view errorCtx) + * { + * e->eval(*this, env, v); + * try { + * if (v.type() != nAttrs) + * throwTypeError("value is %1% while a set was expected", v); + * } catch (Error & e) { + * e.addTrace(pos, errorCtx); + * throw; + * } + * } + * ``` + * + * The second example can be rewritten more concisely, but kept in this + * form to highlight the symmetry. The first option adds more information, + * because whatever caused an error down the line, in the generic eval + * function, will get annotated with the code location that uses and + * required it. The second option is less verbose, but does not provide + * any context at all as to where and why a failing value was required. + * + * Scopes would fix that, by adding context only when --show-trace is + * passed, and keeping the trace terse otherwise. + * + */ + + // Enough indent to align with with the `... ` + // prepended to each element of the trace + auto ellipsisIndent = " "; + + bool frameOnly = false; + if (!einfo.traces.empty()) { + size_t count = 0; + for (const auto & trace : einfo.traces) { + if (trace.hint.str().empty()) continue; + if (frameOnly && !trace.frame) continue; - // lines of code. - if (loc.has_value()) { + if (!showTrace && count > 3) { + oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full trace)" ANSI_NORMAL << "\n"; + break; + } + + count++; + frameOnly = trace.frame; + + oss << "\n" << "… " << trace.hint.str() << "\n"; + + if (trace.pos) { + count++; + + oss << "\n" << ellipsisIndent << ANSI_BLUE << "at " ANSI_WARNING << *trace.pos << ANSI_NORMAL << ":"; + + if (auto loc = trace.pos->getCodeLines()) { + oss << "\n"; + printCodeLines(oss, "", *trace.pos, *loc); + oss << "\n"; + } else + oss << noSource; + } + } + oss << "\n" << prefix; + } + + oss << einfo.msg << "\n"; + + if (einfo.errPos) { + oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *einfo.errPos << ANSI_NORMAL << ":"; + + if (auto loc = einfo.errPos->getCodeLines()) { oss << "\n"; printCodeLines(oss, "", *einfo.errPos, *loc); oss << "\n"; - } + } else + oss << noSource; } auto suggestions = einfo.suggestions.trim(); - if (! suggestions.suggestions.empty()){ + if (!suggestions.suggestions.empty()) { oss << "Did you mean " << suggestions.trim() << "?" << std::endl; } - // traces - if (showTrace && !einfo.traces.empty()) { - for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) { - oss << "\n" << "… " << iter->hint.str() << "\n"; - - if (iter->pos.has_value() && (*iter->pos)) { - auto pos = iter->pos.value(); - oss << "\n"; - printAtPos(pos, oss); - - auto loc = getCodeLines(pos); - if (loc.has_value()) { - oss << "\n"; - printCodeLines(oss, "", pos, *loc); - oss << "\n"; - } - } - } - } - out << indent(prefix, std::string(filterANSIEscapes(prefix, true).size(), ' '), chomp(oss.str())); return out; diff -Nru nix-2.8.0/src/libutil/error.hh nix-2.16.1+dfsg/src/libutil/error.hh --- nix-2.8.0/src/libutil/error.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/error.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,19 @@ #pragma once +/** + * @file + * + * @brief This file defines two main structs/classes used in nix error handling. + * + * ErrorInfo provides a standard payload of error information, with conversion to string + * happening in the logger rather than at the call site. + * + * BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains + * an ErrorInfo. + * + * ErrorInfo structs are sent to the logger as part of an exception, or directly with the + * logError or logWarning macros. + * See libutil/tests/logging.cc for usage examples. + */ #include "suggestions.hh" #include "ref.hh" @@ -26,22 +41,6 @@ namespace nix { -/* - - This file defines two main structs/classes used in nix error handling. - - ErrorInfo provides a standard payload of error information, with conversion to string - happening in the logger rather than at the call site. - - BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains - an ErrorInfo. - - ErrorInfo structs are sent to the logger as part of an exception, or directly with the - logError or logWarning macros. - - See libutil/tests/logging.cc for usage examples. - - */ typedef enum { lvlError = 0, @@ -54,63 +53,53 @@ lvlVomit } Verbosity; -/* adjust Pos::origin bit width when adding stuff here */ -typedef enum { - foFile, - foStdin, - foString -} FileOrigin; - -// the lines of code surrounding an error. +/** + * The lines of code surrounding an error. + */ struct LinesOfCode { std::optional prevLineOfCode; std::optional errLineOfCode; std::optional nextLineOfCode; }; -// ErrPos indicates the location of an error in a nix file. -struct ErrPos { - int line = 0; - int column = 0; - std::string file; - FileOrigin origin; +/** + * An abstract type that represents a location in a source file. + */ +struct AbstractPos +{ + uint32_t line = 0; + uint32_t column = 0; - operator bool() const - { - return line != 0; - } + /** + * Return the contents of the source file. + */ + virtual std::optional getSource() const + { return std::nullopt; }; - // convert from the Pos struct, found in libexpr. - template - ErrPos & operator=(const P & pos) - { - origin = pos.origin; - line = pos.line; - column = pos.column; - // is file symbol null? - if (pos.file.set()) - file = pos.file; - else - file = ""; - return *this; - } + virtual void print(std::ostream & out) const = 0; - template - ErrPos(const P & p) - { - *this = p; - } + std::optional getCodeLines() const; + + virtual ~AbstractPos() = default; }; +std::ostream & operator << (std::ostream & str, const AbstractPos & pos); + +void printCodeLines(std::ostream & out, + const std::string & prefix, + const AbstractPos & errPos, + const LinesOfCode & loc); + struct Trace { - std::optional pos; + std::shared_ptr pos; hintformat hint; + bool frame; }; struct ErrorInfo { Verbosity level; hintformat msg; - std::optional errPos; + std::shared_ptr errPos; std::list traces; Suggestions suggestions; @@ -120,8 +109,10 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace); -/* BaseError should generally not be caught, as it has Interrupted as - a subclass. Catch Error instead. */ +/** + * BaseError should generally not be caught, as it has Interrupted as + * a subclass. Catch Error instead. + */ class BaseError : public std::exception { protected: @@ -133,6 +124,8 @@ public: unsigned int status = 1; // exit status + BaseError(const BaseError &) = default; + template BaseError(unsigned int status, const Args & ... args) : err { .level = lvlError, .msg = hintfmt(args...) } @@ -171,15 +164,22 @@ const std::string & msg() const { return calcWhat(); } const ErrorInfo & info() const { calcWhat(); return err; } + void pushTrace(Trace trace) + { + err.traces.push_front(trace); + } + template - void addTrace(std::optional e, const std::string & fs, const Args & ... args) + void addTrace(std::shared_ptr && e, std::string_view fs, const Args & ... args) { - addTrace(e, hintfmt(fs, args...)); + addTrace(std::move(e), hintfmt(std::string(fs), args...)); } - void addTrace(std::optional e, hintformat hint); + void addTrace(std::shared_ptr && e, hintformat hint, bool frame = false); bool hasTrace() const { return !err.traces.empty(); } + + const ErrorInfo & info() { return err; }; }; #define MakeError(newClass, superClass) \ @@ -199,13 +199,19 @@ int errNo; template - SysError(const Args & ... args) + SysError(int errNo_, const Args & ... args) : Error("") { - errNo = errno; + errNo = errNo_; auto hf = hintfmt(args...); err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } + + template + SysError(const Args & ... args) + : SysError(errno, args ...) + { + } }; } diff -Nru nix-2.8.0/src/libutil/experimental-features.cc nix-2.16.1+dfsg/src/libutil/experimental-features.cc --- nix-2.8.0/src/libutil/experimental-features.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/experimental-features.cc 2023-06-06 10:55:23.000000000 +0000 @@ -5,25 +5,234 @@ namespace nix { -std::map stringifiedXpFeatures = { - { Xp::CaDerivations, "ca-derivations" }, - { Xp::ImpureDerivations, "impure-derivations" }, - { Xp::Flakes, "flakes" }, - { Xp::NixCommand, "nix-command" }, - { Xp::RecursiveNix, "recursive-nix" }, - { Xp::NoUrlLiterals, "no-url-literals" }, - { Xp::FetchClosure, "fetch-closure" }, +struct ExperimentalFeatureDetails +{ + ExperimentalFeature tag; + std::string_view name; + std::string_view description; }; +constexpr std::array xpFeatureDetails = {{ + { + .tag = Xp::CaDerivations, + .name = "ca-derivations", + .description = R"( + Allow derivations to be content-addressed in order to prevent + rebuilds when changes to the derivation do not result in changes to + the derivation's output. See + [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed) + for details. + )", + }, + { + .tag = Xp::ImpureDerivations, + .name = "impure-derivations", + .description = R"( + Allow derivations to produce non-fixed outputs by setting the + `__impure` derivation attribute to `true`. An impure derivation can + have differing outputs each time it is built. + + Example: + + ``` + derivation { + name = "impure"; + builder = /bin/sh; + __impure = true; # mark this derivation as impure + args = [ "-c" "read -n 10 random < /dev/random; echo $random > $out" ]; + system = builtins.currentSystem; + } + ``` + + Each time this derivation is built, it can produce a different + output (as the builder outputs random bytes to `$out`). Impure + derivations also have access to the network, and only fixed-output + or other impure derivations can rely on impure derivations. Finally, + an impure derivation cannot also be + [content-addressed](#xp-feature-ca-derivations). + + This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime). + )", + }, + { + .tag = Xp::Flakes, + .name = "flakes", + .description = R"( + Enable flakes. See the manual entry for [`nix + flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. + )", + }, + { + .tag = Xp::NixCommand, + .name = "nix-command", + .description = R"( + Enable the new `nix` subcommands. See the manual on + [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. + )", + }, + { + .tag = Xp::RecursiveNix, + .name = "recursive-nix", + .description = R"( + Allow derivation builders to call Nix, and thus build derivations + recursively. + + Example: + + ``` + with import {}; + + runCommand "foo" + { + buildInputs = [ nix jq ]; + NIX_PATH = "nixpkgs=${}"; + } + '' + hello=$(nix-build -E '(import {}).hello.overrideDerivation (args: { name = "recursive-hello"; })') + + mkdir -p $out/bin + ln -s $hello/bin/hello $out/bin/hello + '' + ``` + + An important restriction on recursive builders is disallowing + arbitrary substitutions. For example, running + + ``` + nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10 + ``` + + in the above `runCommand` script would be disallowed, as this could + lead to derivations with hidden dependencies or breaking + reproducibility by relying on the current state of the Nix store. An + exception would be if + `/nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10` were + already in the build inputs or built by a previous recursive Nix + call. + )", + }, + { + .tag = Xp::NoUrlLiterals, + .name = "no-url-literals", + .description = R"( + Disallow unquoted URLs as part of the Nix language syntax. The Nix + language allows for URL literals, like so: + + ``` + $ nix repl + Welcome to Nix 2.15.0. Type :? for help. + + nix-repl> http://foo + "http://foo" + ``` + + But enabling this experimental feature will cause the Nix parser to + throw an error when encountering a URL literal: + + ``` + $ nix repl --extra-experimental-features 'no-url-literals' + Welcome to Nix 2.15.0. Type :? for help. + + nix-repl> http://foo + error: URL literals are disabled + + at «string»:1:1: + + 1| http://foo + | ^ + + ``` + + While this is currently an experimental feature, unquoted URLs are + being deprecated and their usage is discouraged. + + The reason is that, as opposed to path literals, URLs have no + special properties that distinguish them from regular strings, URLs + containing parameters have to be quoted anyway, and unquoted URLs + may confuse external tooling. + )", + }, + { + .tag = Xp::FetchClosure, + .name = "fetch-closure", + .description = R"( + Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language. + )", + }, + { + .tag = Xp::ReplFlake, + .name = "repl-flake", + .description = R"( + Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. + )", + }, + { + .tag = Xp::AutoAllocateUids, + .name = "auto-allocate-uids", + .description = R"( + Allows Nix to automatically pick UIDs for builds, rather than creating + `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. + )", + }, + { + .tag = Xp::Cgroups, + .name = "cgroups", + .description = R"( + Allows Nix to execute builds inside cgroups. See + the [`use-cgroups`](#conf-use-cgroups) setting for details. + )", + }, + { + .tag = Xp::DiscardReferences, + .name = "discard-references", + .description = R"( + Allow the use of the [`unsafeDiscardReferences`](@docroot@/language/advanced-attributes.html#adv-attr-unsafeDiscardReferences) attribute in derivations + that use [structured attributes](@docroot@/language/advanced-attributes.html#adv-attr-structuredAttrs). This disables scanning of outputs for + runtime dependencies. + )", + }, + { + .tag = Xp::DaemonTrustOverride, + .name = "daemon-trust-override", + .description = R"( + Allow forcing trusting or not trusting clients with + `nix-daemon`. This is useful for testing, but possibly also + useful for various experiments with `nix-daemon --stdio` + networking. + )", + }, + { + .tag = Xp::DynamicDerivations, + .name = "dynamic-derivations", + .description = R"( + Allow the use of a few things related to dynamic derivations: + + - "text hashing" derivation outputs, so we can build .drv + files. + + - dependencies in derivations on the outputs of + derivations that are themselves derivations outputs. + )", + }, +}}; + +static_assert( + []() constexpr { + for (auto [index, feature] : enumerate(xpFeatureDetails)) + if (index != (size_t)feature.tag) + return false; + return true; + }(), + "array order does not match enum tag order"); + const std::optional parseExperimentalFeature(const std::string_view & name) { using ReverseXpMap = std::map; - static auto reverseXpMap = []() - { + static std::unique_ptr reverseXpMap = []() { auto reverseXpMap = std::make_unique(); - for (auto & [feature, name] : stringifiedXpFeatures) - (*reverseXpMap)[name] = feature; + for (auto & xpFeature : xpFeatureDetails) + (*reverseXpMap)[xpFeature.name] = xpFeature.tag; return reverseXpMap; }(); @@ -33,18 +242,27 @@ return std::nullopt; } -std::string_view showExperimentalFeature(const ExperimentalFeature feature) +std::string_view showExperimentalFeature(const ExperimentalFeature tag) +{ + assert((size_t)tag < xpFeatureDetails.size()); + return xpFeatureDetails[(size_t)tag].name; +} + +nlohmann::json documentExperimentalFeatures() { - return stringifiedXpFeatures.at(feature); + StringMap res; + for (auto & xpFeature : xpFeatureDetails) + res[std::string { xpFeature.name }] = + trim(stripIndentation(xpFeature.description)); + return (nlohmann::json) res; } std::set parseFeatures(const std::set & rawFeatures) { std::set res; - for (auto & rawFeature : rawFeatures) { + for (auto & rawFeature : rawFeatures) if (auto feature = parseExperimentalFeature(rawFeature)) res.insert(*feature); - } return res; } @@ -58,4 +276,20 @@ return str << showExperimentalFeature(feature); } +void to_json(nlohmann::json & j, const ExperimentalFeature & feature) +{ + j = showExperimentalFeature(feature); +} + +void from_json(const nlohmann::json & j, ExperimentalFeature & feature) +{ + const std::string input = j; + const auto parsed = parseExperimentalFeature(input); + + if (parsed.has_value()) + feature = *parsed; + else + throw Error("Unknown experimental feature '%s' in JSON input", input); +} + } diff -Nru nix-2.8.0/src/libutil/experimental-features.hh nix-2.16.1+dfsg/src/libutil/experimental-features.hh --- nix-2.8.0/src/libutil/experimental-features.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/experimental-features.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "comparator.hh" #include "error.hh" @@ -10,9 +11,10 @@ /** * The list of available experimental features. * - * If you update this, don’t forget to also change the map defining their - * string representation in the corresponding `.cc` file. - **/ + * If you update this, don’t forget to also change the map defining + * their string representation and documentation in the corresponding + * `.cc` file as well. + */ enum struct ExperimentalFeature { CaDerivations, @@ -22,6 +24,12 @@ RecursiveNix, NoUrlLiterals, FetchClosure, + ReplFlake, + AutoAllocateUids, + Cgroups, + DiscardReferences, + DaemonTrustOverride, + DynamicDerivations, }; /** @@ -29,26 +37,59 @@ */ using Xp = ExperimentalFeature; +/** + * Parse an experimental feature (enum value) from its name. Experimental + * feature flag names are hyphenated and do not contain spaces. + */ const std::optional parseExperimentalFeature( const std::string_view & name); + +/** + * Show the name of an experimental feature. This is the opposite of + * parseExperimentalFeature(). + */ std::string_view showExperimentalFeature(const ExperimentalFeature); +/** + * Compute the documentation of all experimental features. + * + * See `doc/manual` for how this information is used. + */ +nlohmann::json documentExperimentalFeatures(); + +/** + * Shorthand for `str << showExperimentalFeature(feature)`. + */ std::ostream & operator<<( std::ostream & str, const ExperimentalFeature & feature); /** - * Parse a set of strings to the corresponding set of experimental features, - * ignoring (but warning for) any unkwown feature. + * Parse a set of strings to the corresponding set of experimental + * features, ignoring (but warning for) any unknown feature. */ std::set parseFeatures(const std::set &); +/** + * An experimental feature was required for some (experimental) + * operation, but was not enabled. + */ class MissingExperimentalFeature : public Error { public: + /** + * The experimental feature that was required but not enabled. + */ ExperimentalFeature missingFeature; - MissingExperimentalFeature(ExperimentalFeature); + MissingExperimentalFeature(ExperimentalFeature missingFeature); }; +/** + * Semi-magic conversion to and from json. + * See the nlohmann/json readme for more details. + */ +void to_json(nlohmann::json &, const ExperimentalFeature &); +void from_json(const nlohmann::json &, ExperimentalFeature &); + } diff -Nru nix-2.8.0/src/libutil/filesystem.cc nix-2.16.1+dfsg/src/libutil/filesystem.cc --- nix-2.8.0/src/libutil/filesystem.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/filesystem.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,173 @@ +#include +#include +#include + +#include "finally.hh" +#include "util.hh" +#include "types.hh" + +namespace fs = std::filesystem; + +namespace nix { + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + std::atomic & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + if (includePid) + return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); + else + return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static std::atomic globalCounter = 0; + std::atomic localCounter = 0; + auto & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); + return {std::move(fd), tmpl}; +} + +void createSymlink(const Path & target, const Path & link, + std::optional mtime) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError("creating symlink from '%1%' to '%2%'", link, target); + if (mtime) { + struct timeval times[2]; + times[0].tv_sec = *mtime; + times[0].tv_usec = 0; + times[1].tv_sec = *mtime; + times[1].tv_usec = 0; + if (lutimes(link.c_str(), times)) + throw SysError("setting time of symlink '%s'", link); + } +} + +void replaceSymlink(const Path & target, const Path & link, + std::optional mtime) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp, mtime); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + renameFile(tmp, link); + + break; + } +} + +void setWriteTime(const fs::path & p, const struct stat & st) +{ + struct timeval times[2]; + times[0] = { + .tv_sec = st.st_atime, + .tv_usec = 0, + }; + times[1] = { + .tv_sec = st.st_mtime, + .tv_usec = 0, + }; + if (lutimes(p.c_str(), times) != 0) + throw SysError("changing modification time of '%s'", p); +} + +void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) +{ + // TODO: Rewrite the `is_*` to use `symlink_status()` + auto statOfFrom = lstat(from.path().c_str()); + auto fromStatus = from.symlink_status(); + + // Mark the directory as writable so that we can delete its children + if (andDelete && fs::is_directory(fromStatus)) { + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from.path())) { + copy(entry, to / entry.path().filename(), andDelete); + } + } else { + throw Error("file '%s' has an unsupported type", from.path()); + } + + setWriteTime(to, statOfFrom); + if (andDelete) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from.path()); + } +} + +void renameFile(const Path & oldName, const Path & newName) +{ + fs::rename(oldName, newName); +} + +void moveFile(const Path & oldName, const Path & newName) +{ + try { + renameFile(oldName, newName); + } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; + if (e.code().value() == EXDEV) { + fs::remove(newPath); + warn("Can’t rename %s as %s, copying instead", oldName, newName); + copy(fs::directory_entry(oldPath), tempCopyTarget, true); + renameFile(tempCopyTarget, newPath); + } + } +} + +} diff -Nru nix-2.8.0/src/libutil/finally.hh nix-2.16.1+dfsg/src/libutil/finally.hh --- nix-2.8.0/src/libutil/finally.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/finally.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,9 @@ #pragma once +///@file -/* A trivial class to run a function at the end of a scope. */ +/** + * A trivial class to run a function at the end of a scope. + */ template class Finally { diff -Nru nix-2.8.0/src/libutil/fmt.cc nix-2.16.1+dfsg/src/libutil/fmt.cc --- nix-2.8.0/src/libutil/fmt.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/fmt.cc 1970-01-01 00:00:00.000000000 +0000 @@ -1,46 +0,0 @@ -#include "fmt.hh" - -#include - -namespace nix { - -std::string hiliteMatches( - std::string_view s, - std::vector matches, - std::string_view prefix, - std::string_view postfix) -{ - // Avoid copy on zero matches - if (matches.size() == 0) - return (std::string) s; - - std::sort(matches.begin(), matches.end(), [](const auto & a, const auto & b) { - return a.position() < b.position(); - }); - - std::string out; - ssize_t last_end = 0; - - for (auto it = matches.begin(); it != matches.end();) { - auto m = *it; - size_t start = m.position(); - out.append(s.substr(last_end, m.position() - last_end)); - // Merge continous matches - ssize_t end = start + m.length(); - while (++it != matches.end() && (*it).position() <= end) { - auto n = *it; - ssize_t nend = start + (n.position() - start + n.length()); - if (nend > end) - end = nend; - } - out.append(prefix); - out.append(s.substr(start, end - start)); - out.append(postfix); - last_end = end; - } - - out.append(s.substr(last_end)); - return out; -} - -} diff -Nru nix-2.8.0/src/libutil/fmt.hh nix-2.16.1+dfsg/src/libutil/fmt.hh --- nix-2.8.0/src/libutil/fmt.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/fmt.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,38 +1,33 @@ #pragma once +///@file #include #include -#include #include "ansicolor.hh" namespace nix { -/* Inherit some names from other namespaces for convenience. */ +/** + * Inherit some names from other namespaces for convenience. + */ using boost::format; -/* A variadic template that does nothing. Useful to call a function - for all variadic arguments but ignoring the result. */ +/** + * A variadic template that does nothing. Useful to call a function + * for all variadic arguments but ignoring the result. + */ struct nop { template nop(T...) {} }; -struct FormatOrString -{ - std::string s; - FormatOrString(std::string s) : s(std::move(s)) { }; - template - FormatOrString(const F & f) : s(f.str()) { }; - FormatOrString(const char * s) : s(s) { }; -}; - - -/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is - equivalent to ‘boost::format(format) % a_0 % ... % - ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion - takes place). */ - +/** + * A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is + * equivalent to ‘boost::format(format) % a_0 % ... % + * ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion + * takes place). + */ template inline void formatHelper(F & f) { @@ -54,11 +49,6 @@ return s; } -inline std::string fmt(const FormatOrString & fs) -{ - return fs.s; -} - template inline std::string fmt(const std::string & fs, const Args & ... args) { @@ -149,21 +139,10 @@ return f; } -inline hintformat hintfmt(std::string plain_string) +inline hintformat hintfmt(const std::string & plain_string) { // we won't be receiving any args in this case, so just print the original string return hintfmt("%s", normaltxt(plain_string)); } -/* Highlight all the given matches in the given string `s` by wrapping - them between `prefix` and `postfix`. - - If some matches overlap, then their union will be wrapped rather - than the individual matches. */ -std::string hiliteMatches( - std::string_view s, - std::vector matches, - std::string_view prefix, - std::string_view postfix); - } diff -Nru nix-2.8.0/src/libutil/git.cc nix-2.16.1+dfsg/src/libutil/git.cc --- nix-2.8.0/src/libutil/git.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/git.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,25 @@ +#include "git.hh" + +#include + +namespace nix { +namespace git { + +std::optional parseLsRemoteLine(std::string_view line) +{ + const static std::regex line_regex("^(ref: *)?([^\\s]+)(?:\\t+(.*))?$"); + std::match_results match; + if (!std::regex_match(line.cbegin(), line.cend(), match, line_regex)) + return std::nullopt; + + return LsRemoteRefLine { + .kind = match[1].length() == 0 + ? LsRemoteRefLine::Kind::Object + : LsRemoteRefLine::Kind::Symbolic, + .target = match[2], + .reference = match[3].length() == 0 ? std::nullopt : std::optional{ match[3] } + }; +} + +} +} diff -Nru nix-2.8.0/src/libutil/git.hh nix-2.16.1+dfsg/src/libutil/git.hh --- nix-2.8.0/src/libutil/git.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/git.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,43 @@ +#pragma once +///@file + +#include +#include +#include + +namespace nix { + +namespace git { + +/** + * A line from the output of `git ls-remote --symref`. + * + * These can be of two kinds: + * + * - Symbolic references of the form + * + * ref: {target} {reference} + * + * where {target} is itself a reference and {reference} is optional + * + * - Object references of the form + * + * {target} {reference} + * + * where {target} is a commit id and {reference} is mandatory + */ +struct LsRemoteRefLine { + enum struct Kind { + Symbolic, + Object + }; + Kind kind; + std::string target; + std::optional reference; +}; + +std::optional parseLsRemoteLine(std::string_view line); + +} + +} diff -Nru nix-2.8.0/src/libutil/hash.cc nix-2.16.1+dfsg/src/libutil/hash.cc --- nix-2.8.0/src/libutil/hash.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/hash.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -16,7 +17,6 @@ namespace nix { - static size_t regularHashSize(HashType type) { switch (type) { case htMD5: return md5HashSize; @@ -71,12 +71,13 @@ static std::string printHash16(const Hash & hash) { - char buf[hash.hashSize * 2]; + std::string buf; + buf.reserve(hash.hashSize * 2); for (unsigned int i = 0; i < hash.hashSize; i++) { - buf[i * 2] = base16Chars[hash.hash[i] >> 4]; - buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; + buf.push_back(base16Chars[hash.hash[i] >> 4]); + buf.push_back(base16Chars[hash.hash[i] & 0x0f]); } - return std::string(buf, hash.hashSize * 2); + return buf; } @@ -130,7 +131,7 @@ break; case Base64: case SRI: - s += base64Encode(std::string((const char *) hash, hashSize)); + s += base64Encode(std::string_view((const char *) hash, hashSize)); break; } return s; @@ -342,7 +343,7 @@ delete ctx; } -void HashSink::write(std::string_view data) +void HashSink::writeUnbuffered(std::string_view data) { bytes += data.size(); update(ht, *ctx, data); @@ -403,7 +404,7 @@ throw UsageError("unknown hash algorithm '%1%'", s); } -std::string printHashType(HashType ht) +std::string_view printHashType(HashType ht) { switch (ht) { case htMD5: return "md5"; diff -Nru nix-2.8.0/src/libutil/hash.hh nix-2.16.1+dfsg/src/libutil/hash.hh --- nix-2.8.0/src/libutil/hash.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/hash.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "serialise.hh" @@ -33,62 +34,86 @@ HashType type; - /* Create a zero-filled hash object. */ + /** + * Create a zero-filled hash object. + */ Hash(HashType type); - /* Parse the hash from a string representation in the format - "[:]" or "-" (a - Subresource Integrity hash expression). If the 'type' argument - is not present, then the hash type must be specified in the - string. */ + /** + * Parse the hash from a string representation in the format + * "[:]" or "-" (a + * Subresource Integrity hash expression). If the 'type' argument + * is not present, then the hash type must be specified in the + * string. + */ static Hash parseAny(std::string_view s, std::optional type); - /* Parse a hash from a string representation like the above, except the - type prefix is mandatory is there is no separate arguement. */ + /** + * Parse a hash from a string representation like the above, except the + * type prefix is mandatory is there is no separate arguement. + */ static Hash parseAnyPrefixed(std::string_view s); - /* Parse a plain hash that musst not have any prefix indicating the type. - The type is passed in to disambiguate. */ + /** + * Parse a plain hash that musst not have any prefix indicating the type. + * The type is passed in to disambiguate. + */ static Hash parseNonSRIUnprefixed(std::string_view s, HashType type); static Hash parseSRI(std::string_view original); private: - /* The type must be provided, the string view must not include - prefix. `isSRI` helps disambigate the various base-* encodings. */ + /** + * The type must be provided, the string view must not include + * prefix. `isSRI` helps disambigate the various base-* encodings. + */ Hash(std::string_view s, HashType type, bool isSRI); public: - /* Check whether two hash are equal. */ + /** + * Check whether two hash are equal. + */ bool operator == (const Hash & h2) const; - /* Check whether two hash are not equal. */ + /** + * Check whether two hash are not equal. + */ bool operator != (const Hash & h2) const; - /* For sorting. */ + /** + * For sorting. + */ bool operator < (const Hash & h) const; - /* Returns the length of a base-16 representation of this hash. */ + /** + * Returns the length of a base-16 representation of this hash. + */ size_t base16Len() const { return hashSize * 2; } - /* Returns the length of a base-32 representation of this hash. */ + /** + * Returns the length of a base-32 representation of this hash. + */ size_t base32Len() const { return (hashSize * 8 - 1) / 5 + 1; } - /* Returns the length of a base-64 representation of this hash. */ + /** + * Returns the length of a base-64 representation of this hash. + */ size_t base64Len() const { return ((4 * hashSize / 3) + 3) & ~3; } - /* Return a string representation of the hash, in base-16, base-32 - or base-64. By default, this is prefixed by the hash type - (e.g. "sha256:"). */ + /** + * Return a string representation of the hash, in base-16, base-32 + * or base-64. By default, this is prefixed by the hash type + * (e.g. "sha256:"). + */ std::string to_string(Base base, bool includeType) const; std::string gitRev() const @@ -104,36 +129,54 @@ static Hash dummy; }; -/* Helper that defaults empty hashes to the 0 hash. */ +/** + * Helper that defaults empty hashes to the 0 hash. + */ Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht); -/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ +/** + * Print a hash in base-16 if it's MD5, or base-32 otherwise. + */ std::string printHash16or32(const Hash & hash); -/* Compute the hash of the given string. */ +/** + * Compute the hash of the given string. + */ Hash hashString(HashType ht, std::string_view s); -/* Compute the hash of the given file. */ +/** + * Compute the hash of the given file. + */ Hash hashFile(HashType ht, const Path & path); -/* Compute the hash of the given path. The hash is defined as - (essentially) hashString(ht, dumpPath(path)). */ +/** + * Compute the hash of the given path. The hash is defined as + * (essentially) hashString(ht, dumpPath(path)). + */ typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); -/* Compress a hash to the specified number of bytes by cyclically - XORing bytes together. */ +/** + * Compress a hash to the specified number of bytes by cyclically + * XORing bytes together. + */ Hash compressHash(const Hash & hash, unsigned int newSize); -/* Parse a string representing a hash type. */ +/** + * Parse a string representing a hash type. + */ HashType parseHashType(std::string_view s); -/* Will return nothing on parse error */ +/** + * Will return nothing on parse error + */ std::optional parseHashTypeOpt(std::string_view s); -/* And the reverse. */ -std::string printHashType(HashType ht); +/** + * And the reverse. + */ +std::string_view printHashType(HashType ht); union Ctx; @@ -154,7 +197,7 @@ HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - void write(std::string_view data) override; + void writeUnbuffered(std::string_view data) override; HashResult finish() override; HashResult currentHash(); }; diff -Nru nix-2.8.0/src/libutil/hilite.cc nix-2.16.1+dfsg/src/libutil/hilite.cc --- nix-2.8.0/src/libutil/hilite.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/hilite.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,44 @@ +#include "hilite.hh" + +namespace nix { + +std::string hiliteMatches( + std::string_view s, + std::vector matches, + std::string_view prefix, + std::string_view postfix) +{ + // Avoid extra work on zero matches + if (matches.size() == 0) + return std::string(s); + + std::sort(matches.begin(), matches.end(), [](const auto & a, const auto & b) { + return a.position() < b.position(); + }); + + std::string out; + ssize_t last_end = 0; + + for (auto it = matches.begin(); it != matches.end();) { + auto m = *it; + size_t start = m.position(); + out.append(s.substr(last_end, m.position() - last_end)); + // Merge continous matches + ssize_t end = start + m.length(); + while (++it != matches.end() && (*it).position() <= end) { + auto n = *it; + ssize_t nend = start + (n.position() - start + n.length()); + if (nend > end) + end = nend; + } + out.append(prefix); + out.append(s.substr(start, end - start)); + out.append(postfix); + last_end = end; + } + + out.append(s.substr(last_end)); + return out; +} + +} diff -Nru nix-2.8.0/src/libutil/hilite.hh nix-2.16.1+dfsg/src/libutil/hilite.hh --- nix-2.8.0/src/libutil/hilite.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/hilite.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,23 @@ +#pragma once +///@file + +#include +#include +#include + +namespace nix { + +/** + * Highlight all the given matches in the given string `s` by wrapping + * them between `prefix` and `postfix`. + * + * If some matches overlap, then their union will be wrapped rather + * than the individual matches. + */ +std::string hiliteMatches( + std::string_view s, + std::vector matches, + std::string_view prefix, + std::string_view postfix); + +} diff -Nru nix-2.8.0/src/libutil/json.cc nix-2.16.1+dfsg/src/libutil/json.cc --- nix-2.8.0/src/libutil/json.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/json.cc 1970-01-01 00:00:00.000000000 +0000 @@ -1,201 +0,0 @@ -#include "json.hh" - -#include -#include - -namespace nix { - -void toJSON(std::ostream & str, const char * start, const char * end) -{ - constexpr size_t BUF_SIZE = 4096; - char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts - size_t bufPos = 0; - - const auto flush = [&] { - str.write(buf, bufPos); - bufPos = 0; - }; - const auto put = [&] (char c) { - buf[bufPos++] = c; - }; - - put('"'); - for (auto i = start; i != end; i++) { - if (bufPos >= BUF_SIZE) flush(); - if (*i == '\"' || *i == '\\') { put('\\'); put(*i); } - else if (*i == '\n') { put('\\'); put('n'); } - else if (*i == '\r') { put('\\'); put('r'); } - else if (*i == '\t') { put('\\'); put('t'); } - else if (*i >= 0 && *i < 32) { - const char hex[17] = "0123456789abcdef"; - put('\\'); - put('u'); - put(hex[(uint16_t(*i) >> 12) & 0xf]); - put(hex[(uint16_t(*i) >> 8) & 0xf]); - put(hex[(uint16_t(*i) >> 4) & 0xf]); - put(hex[(uint16_t(*i) >> 0) & 0xf]); - } - else put(*i); - } - put('"'); - flush(); -} - -void toJSON(std::ostream & str, const char * s) -{ - if (!s) str << "null"; else toJSON(str, s, s + strlen(s)); -} - -template<> void toJSON(std::ostream & str, const int & n) { str << n; } -template<> void toJSON(std::ostream & str, const unsigned int & n) { str << n; } -template<> void toJSON(std::ostream & str, const long & n) { str << n; } -template<> void toJSON(std::ostream & str, const unsigned long & n) { str << n; } -template<> void toJSON(std::ostream & str, const long long & n) { str << n; } -template<> void toJSON(std::ostream & str, const unsigned long long & n) { str << n; } -template<> void toJSON(std::ostream & str, const float & n) { str << n; } -template<> void toJSON(std::ostream & str, const double & n) { str << n; } - -template<> void toJSON(std::ostream & str, const std::string & s) -{ - toJSON(str, s.c_str(), s.c_str() + s.size()); -} - -template<> void toJSON(std::ostream & str, const bool & b) -{ - str << (b ? "true" : "false"); -} - -template<> void toJSON(std::ostream & str, const std::nullptr_t & b) -{ - str << "null"; -} - -JSONWriter::JSONWriter(std::ostream & str, bool indent) - : state(new JSONState(str, indent)) -{ - state->stack++; -} - -JSONWriter::JSONWriter(JSONState * state) - : state(state) -{ - state->stack++; -} - -JSONWriter::~JSONWriter() -{ - if (state) { - assertActive(); - state->stack--; - if (state->stack == 0) delete state; - } -} - -void JSONWriter::comma() -{ - assertActive(); - if (first) { - first = false; - } else { - state->str << ','; - } - if (state->indent) indent(); -} - -void JSONWriter::indent() -{ - state->str << '\n' << std::string(state->depth * 2, ' '); -} - -void JSONList::open() -{ - state->depth++; - state->str << '['; -} - -JSONList::~JSONList() -{ - state->depth--; - if (state->indent && !first) indent(); - state->str << "]"; -} - -JSONList JSONList::list() -{ - comma(); - return JSONList(state); -} - -JSONObject JSONList::object() -{ - comma(); - return JSONObject(state); -} - -JSONPlaceholder JSONList::placeholder() -{ - comma(); - return JSONPlaceholder(state); -} - -void JSONObject::open() -{ - state->depth++; - state->str << '{'; -} - -JSONObject::~JSONObject() -{ - if (state) { - state->depth--; - if (state->indent && !first) indent(); - state->str << "}"; - } -} - -void JSONObject::attr(const std::string & s) -{ - comma(); - toJSON(state->str, s); - state->str << ':'; - if (state->indent) state->str << ' '; -} - -JSONList JSONObject::list(const std::string & name) -{ - attr(name); - return JSONList(state); -} - -JSONObject JSONObject::object(const std::string & name) -{ - attr(name); - return JSONObject(state); -} - -JSONPlaceholder JSONObject::placeholder(const std::string & name) -{ - attr(name); - return JSONPlaceholder(state); -} - -JSONList JSONPlaceholder::list() -{ - assertValid(); - first = false; - return JSONList(state); -} - -JSONObject JSONPlaceholder::object() -{ - assertValid(); - first = false; - return JSONObject(state); -} - -JSONPlaceholder::~JSONPlaceholder() -{ - assert(!first || std::uncaught_exceptions()); -} - -} diff -Nru nix-2.8.0/src/libutil/json.hh nix-2.16.1+dfsg/src/libutil/json.hh --- nix-2.8.0/src/libutil/json.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/json.hh 1970-01-01 00:00:00.000000000 +0000 @@ -1,186 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace nix { - -void toJSON(std::ostream & str, const char * start, const char * end); -void toJSON(std::ostream & str, const char * s); - -template -void toJSON(std::ostream & str, const T & n); - -class JSONWriter -{ -protected: - - struct JSONState - { - std::ostream & str; - bool indent; - size_t depth = 0; - size_t stack = 0; - JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { } - ~JSONState() - { - assert(stack == 0); - } - }; - - JSONState * state; - - bool first = true; - - JSONWriter(std::ostream & str, bool indent); - - JSONWriter(JSONState * state); - - ~JSONWriter(); - - void assertActive() - { - assert(state->stack != 0); - } - - void comma(); - - void indent(); -}; - -class JSONObject; -class JSONPlaceholder; - -class JSONList : JSONWriter -{ -private: - - friend class JSONObject; - friend class JSONPlaceholder; - - void open(); - - JSONList(JSONState * state) - : JSONWriter(state) - { - open(); - } - -public: - - JSONList(std::ostream & str, bool indent = false) - : JSONWriter(str, indent) - { - open(); - } - - ~JSONList(); - - template - JSONList & elem(const T & v) - { - comma(); - toJSON(state->str, v); - return *this; - } - - JSONList list(); - - JSONObject object(); - - JSONPlaceholder placeholder(); -}; - -class JSONObject : JSONWriter -{ -private: - - friend class JSONList; - friend class JSONPlaceholder; - - void open(); - - JSONObject(JSONState * state) - : JSONWriter(state) - { - open(); - } - - void attr(const std::string & s); - -public: - - JSONObject(std::ostream & str, bool indent = false) - : JSONWriter(str, indent) - { - open(); - } - - JSONObject(const JSONObject & obj) = delete; - - JSONObject(JSONObject && obj) - : JSONWriter(obj.state) - { - obj.state = 0; - } - - ~JSONObject(); - - template - JSONObject & attr(const std::string & name, const T & v) - { - attr(name); - toJSON(state->str, v); - return *this; - } - - JSONList list(const std::string & name); - - JSONObject object(const std::string & name); - - JSONPlaceholder placeholder(const std::string & name); -}; - -class JSONPlaceholder : JSONWriter -{ - -private: - - friend class JSONList; - friend class JSONObject; - - JSONPlaceholder(JSONState * state) - : JSONWriter(state) - { - } - - void assertValid() - { - assertActive(); - assert(first); - } - -public: - - JSONPlaceholder(std::ostream & str, bool indent = false) - : JSONWriter(str, indent) - { - } - - ~JSONPlaceholder(); - - template - void write(const T & v) - { - assertValid(); - first = false; - toJSON(state->str, v); - } - - JSONList list(); - - JSONObject object(); -}; - -} diff -Nru nix-2.8.0/src/libutil/json-impls.hh nix-2.16.1+dfsg/src/libutil/json-impls.hh --- nix-2.8.0/src/libutil/json-impls.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/json-impls.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,15 @@ +#pragma once +///@file + +#include "nlohmann/json_fwd.hpp" + +// Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types +#define JSON_IMPL(TYPE) \ + namespace nlohmann { \ + using namespace nix; \ + template <> \ + struct adl_serializer { \ + static TYPE from_json(const json & json); \ + static void to_json(json & json, TYPE t); \ + }; \ + } diff -Nru nix-2.8.0/src/libutil/json-utils.hh nix-2.16.1+dfsg/src/libutil/json-utils.hh --- nix-2.8.0/src/libutil/json-utils.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/json-utils.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,22 @@ +#pragma once +///@file + +#include + +namespace nix { + +const nlohmann::json * get(const nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +nlohmann::json * get(nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +} diff -Nru nix-2.8.0/src/libutil/logging.cc nix-2.16.1+dfsg/src/libutil/logging.cc --- nix-2.8.0/src/libutil/logging.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/logging.cc 2023-06-06 10:55:23.000000000 +0000 @@ -32,7 +32,8 @@ void Logger::writeToStdout(std::string_view s) { - std::cout << s << "\n"; + writeFull(STDOUT_FILENO, s); + writeFull(STDOUT_FILENO, "\n"); } class SimpleLogger : public Logger @@ -53,7 +54,7 @@ return printBuildLogs; } - void log(Verbosity lvl, const FormatOrString & fs) override + void log(Verbosity lvl, std::string_view s) override { if (lvl > verbosity) return; @@ -64,14 +65,15 @@ switch (lvl) { case lvlError: c = '3'; break; case lvlWarn: c = '4'; break; - case lvlInfo: c = '5'; break; + case lvlNotice: case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; - default: c = '7'; + case lvlDebug: case lvlVomit: c = '7'; + default: c = '7'; break; // should not happen, and missing enum case is reported by -Werror=switch-enum } prefix = std::string("<") + c + ">"; } - writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); + writeToStderr(prefix + filterANSIEscapes(s, !tty) + "\n"); } void logEI(const ErrorInfo & ei) override @@ -84,7 +86,7 @@ void startActivity(ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, ActivityId parent) - override + override { if (lvl <= verbosity && !s.empty()) log(lvl, s + "..."); @@ -105,14 +107,6 @@ Verbosity verbosity = lvlInfo; -void warnOnce(bool & haveWarned, const FormatOrString & fs) -{ - if (!haveWarned) { - warn(fs.s); - haveWarned = true; - } -} - void writeToStderr(std::string_view s) { try { @@ -130,15 +124,30 @@ return new SimpleLogger(printBuildLogs); } -std::atomic nextId{(uint64_t) getpid() << 32}; +std::atomic nextId{0}; Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s, const Logger::Fields & fields, ActivityId parent) - : logger(logger), id(nextId++) + : logger(logger), id(nextId++ + (((uint64_t) getpid()) << 32)) { logger.startActivity(id, lvl, type, s, fields, parent); } +void to_json(nlohmann::json & json, std::shared_ptr pos) +{ + if (pos) { + json["line"] = pos->line; + json["column"] = pos->column; + std::ostringstream str; + pos->print(str); + json["file"] = str.str(); + } else { + json["line"] = nullptr; + json["column"] = nullptr; + json["file"] = nullptr; + } +} + struct JSONLogger : Logger { Logger & prevLogger; @@ -166,12 +175,12 @@ prevLogger.log(lvlError, "@nix " + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace)); } - void log(Verbosity lvl, const FormatOrString & fs) override + void log(Verbosity lvl, std::string_view s) override { nlohmann::json json; json["action"] = "msg"; json["level"] = lvl; - json["msg"] = fs.s; + json["msg"] = s; write(json); } @@ -185,27 +194,14 @@ json["level"] = ei.level; json["msg"] = oss.str(); json["raw_msg"] = ei.msg.str(); - - if (ei.errPos.has_value() && (*ei.errPos)) { - json["line"] = ei.errPos->line; - json["column"] = ei.errPos->column; - json["file"] = ei.errPos->file; - } else { - json["line"] = nullptr; - json["column"] = nullptr; - json["file"] = nullptr; - } + to_json(json, ei.errPos); if (loggerSettings.showTrace.get() && !ei.traces.empty()) { nlohmann::json traces = nlohmann::json::array(); for (auto iter = ei.traces.rbegin(); iter != ei.traces.rend(); ++iter) { nlohmann::json stackFrame; stackFrame["raw_msg"] = iter->hint.str(); - if (iter->pos.has_value() && (*iter->pos)) { - stackFrame["line"] = iter->pos->line; - stackFrame["column"] = iter->pos->column; - stackFrame["file"] = iter->pos->file; - } + to_json(stackFrame, iter->pos); traces.push_back(stackFrame); } diff -Nru nix-2.8.0/src/libutil/logging.hh nix-2.16.1+dfsg/src/libutil/logging.hh --- nix-2.8.0/src/libutil/logging.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/logging.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "error.hh" @@ -72,17 +73,20 @@ virtual void stop() { }; + virtual void pause() { }; + virtual void resume() { }; + // Whether the logger prints the whole build log virtual bool isVerbose() { return false; } - virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; + virtual void log(Verbosity lvl, std::string_view s) = 0; - void log(const FormatOrString & fs) + void log(std::string_view s) { - log(lvlInfo, fs); + log(lvlInfo, s); } - virtual void logEI(const ErrorInfo &ei) = 0; + virtual void logEI(const ErrorInfo & ei) = 0; void logEI(Verbosity lvl, ErrorInfo ei) { @@ -102,15 +106,16 @@ virtual void writeToStdout(std::string_view s); template - inline void cout(const std::string & fs, const Args & ... args) + inline void cout(const Args & ... args) { - boost::format f(fs); - formatHelper(f, args...); - writeToStdout(f.str()); + writeToStdout(fmt(args...)); } virtual std::optional ask(std::string_view s) { return {}; } + + virtual void setPrintBuildLogs(bool printBuildLogs) + { } }; ActivityId getCurActivity(); @@ -178,12 +183,17 @@ const Activity & act, std::map & activities, bool trusted); -extern Verbosity verbosity; /* suppress msgs > this */ - -/* Print a message with the standard ErrorInfo format. - In general, use these 'log' macros for reporting problems that may require user - intervention or that need more explanation. Use the 'print' macros for more - lightweight status messages. */ +/** + * suppress msgs > this + */ +extern Verbosity verbosity; + +/** + * Print a message with the standard ErrorInfo format. + * In general, use these 'log' macros for reporting problems that may require user + * intervention or that need more explanation. Use the 'print' macros for more + * lightweight status messages. + */ #define logErrorInfo(level, errorInfo...) \ do { \ if ((level) <= nix::verbosity) { \ @@ -194,9 +204,11 @@ #define logError(errorInfo...) logErrorInfo(lvlError, errorInfo) #define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo) -/* Print a string message if the current log level is at least the specified - level. Note that this has to be implemented as a macro to ensure that the - arguments are evaluated lazily. */ +/** + * Print a string message if the current log level is at least the specified + * level. Note that this has to be implemented as a macro to ensure that the + * arguments are evaluated lazily. + */ #define printMsgUsing(loggerParam, level, args...) \ do { \ auto __lvl = level; \ @@ -213,7 +225,9 @@ #define debug(args...) printMsg(lvlDebug, args) #define vomit(args...) printMsg(lvlVomit, args) -/* if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. */ +/** + * if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. + */ template inline void warn(const std::string & fs, const Args & ... args) { @@ -222,7 +236,11 @@ logger->warn(f.str()); } -void warnOnce(bool & haveWarned, const FormatOrString & fs); +#define warnOnce(haveWarned, args...) \ + if (!haveWarned) { \ + haveWarned = true; \ + warn(args); \ + } void writeToStderr(std::string_view s); diff -Nru nix-2.8.0/src/libutil/lru-cache.hh nix-2.16.1+dfsg/src/libutil/lru-cache.hh --- nix-2.8.0/src/libutil/lru-cache.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/lru-cache.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,7 +8,9 @@ namespace nix { -/* A simple least-recently used cache. Not thread-safe. */ +/** + * A simple least-recently used cache. Not thread-safe. + */ template class LRUCache { @@ -31,7 +34,9 @@ LRUCache(size_t capacity) : capacity(capacity) { } - /* Insert or upsert an item in the cache. */ + /** + * Insert or upsert an item in the cache. + */ void upsert(const Key & key, const Value & value) { if (capacity == 0) return; @@ -39,7 +44,9 @@ erase(key); if (data.size() >= capacity) { - /* Retire the oldest item. */ + /** + * Retire the oldest item. + */ auto oldest = lru.begin(); data.erase(*oldest); lru.erase(oldest); @@ -63,14 +70,18 @@ return true; } - /* Look up an item in the cache. If it exists, it becomes the most - recently used item. */ + /** + * Look up an item in the cache. If it exists, it becomes the most + * recently used item. + * */ std::optional get(const Key & key) { auto i = data.find(key); if (i == data.end()) return {}; - /* Move this item to the back of the LRU list. */ + /** + * Move this item to the back of the LRU list. + */ lru.erase(i->second.first.it); auto j = lru.insert(lru.end(), i); i->second.first.it = j; diff -Nru nix-2.8.0/src/libutil/monitor-fd.hh nix-2.16.1+dfsg/src/libutil/monitor-fd.hh --- nix-2.8.0/src/libutil/monitor-fd.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/monitor-fd.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -22,27 +23,38 @@ { thread = std::thread([fd]() { while (true) { - /* Wait indefinitely until a POLLHUP occurs. */ - struct pollfd fds[1]; - fds[0].fd = fd; - /* This shouldn't be necessary, but macOS doesn't seem to - like a zeroed out events field. - See rdar://37537852. - */ - fds[0].events = POLLHUP; - auto count = poll(fds, 1, -1); - if (count == -1) abort(); // can't happen - /* This shouldn't happen, but can on macOS due to a bug. - See rdar://37550628. - - This may eventually need a delay or further - coordination with the main thread if spinning proves - too harmful. - */ - if (count == 0) continue; - assert(fds[0].revents & POLLHUP); - triggerInterrupt(); - break; + /* Wait indefinitely until a POLLHUP occurs. */ + struct pollfd fds[1]; + fds[0].fd = fd; + /* Polling for no specific events (i.e. just waiting + for an error/hangup) doesn't work on macOS + anymore. So wait for read events and ignore + them. */ + fds[0].events = + #ifdef __APPLE__ + POLLRDNORM + #else + 0 + #endif + ; + auto count = poll(fds, 1, -1); + if (count == -1) abort(); // can't happen + /* This shouldn't happen, but can on macOS due to a bug. + See rdar://37550628. + + This may eventually need a delay or further + coordination with the main thread if spinning proves + too harmful. + */ + if (count == 0) continue; + if (fds[0].revents & POLLHUP) { + triggerInterrupt(); + break; + } + /* This will only happen on macOS. We sleep a bit to + avoid waking up too often if the client is sending + input. */ + sleep(1); } }); }; diff -Nru nix-2.8.0/src/libutil/namespaces.cc nix-2.16.1+dfsg/src/libutil/namespaces.cc --- nix-2.8.0/src/libutil/namespaces.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/namespaces.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,97 @@ +#if __linux__ + +#include "namespaces.hh" +#include "util.hh" +#include "finally.hh" + +#include + +namespace nix { + +bool userNamespacesSupported() +{ + static auto res = [&]() -> bool + { + if (!pathExists("/proc/self/ns/user")) { + debug("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y"); + return false; + } + + Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; + if (!pathExists(maxUserNamespaces) || + trim(readFile(maxUserNamespaces)) == "0") + { + debug("user namespaces appear to be disabled; check '/proc/sys/user/max_user_namespaces'"); + return false; + } + + Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; + if (pathExists(procSysKernelUnprivilegedUsernsClone) + && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") + { + debug("user namespaces appear to be disabled; check '/proc/sys/kernel/unprivileged_userns_clone'"); + return false; + } + + try { + Pid pid = startProcess([&]() + { + _exit(0); + }, { + .cloneFlags = CLONE_NEWUSER + }); + + auto r = pid.wait(); + assert(!r); + } catch (SysError & e) { + debug("user namespaces do not work on this system: %s", e.msg()); + return false; + } + + return true; + }(); + return res; +} + +bool mountAndPidNamespacesSupported() +{ + static auto res = [&]() -> bool + { + try { + + Pid pid = startProcess([&]() + { + /* Make sure we don't remount the parent's /proc. */ + if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) + _exit(1); + + /* Test whether we can remount /proc. The kernel disallows + this if /proc is not fully visible, i.e. if there are + filesystems mounted on top of files inside /proc. See + https://lore.kernel.org/lkml/87tvsrjai0.fsf@xmission.com/T/. */ + if (mount("none", "/proc", "proc", 0, 0) == -1) + _exit(2); + + _exit(0); + }, { + .cloneFlags = CLONE_NEWNS | CLONE_NEWPID | (userNamespacesSupported() ? CLONE_NEWUSER : 0) + }); + + if (pid.wait()) { + debug("PID namespaces do not work on this system: cannot remount /proc"); + return false; + } + + } catch (SysError & e) { + debug("mount namespaces do not work on this system: %s", e.msg()); + return false; + } + + return true; + }(); + return res; +} + +} + +#endif diff -Nru nix-2.8.0/src/libutil/namespaces.hh nix-2.16.1+dfsg/src/libutil/namespaces.hh --- nix-2.8.0/src/libutil/namespaces.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/namespaces.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,14 @@ +#pragma once +///@file + +namespace nix { + +#if __linux__ + +bool userNamespacesSupported(); + +bool mountAndPidNamespacesSupported(); + +#endif + +} diff -Nru nix-2.8.0/src/libutil/pool.hh nix-2.16.1+dfsg/src/libutil/pool.hh --- nix-2.8.0/src/libutil/pool.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/pool.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -11,33 +12,37 @@ namespace nix { -/* This template class implements a simple pool manager of resources - of some type R, such as database connections. It is used as - follows: - - class Connection { ... }; - - Pool pool; - - { - auto conn(pool.get()); - conn->exec("select ..."); - } - - Here, the Connection object referenced by ‘conn’ is automatically - returned to the pool when ‘conn’ goes out of scope. -*/ - +/** + * This template class implements a simple pool manager of resources + * of some type R, such as database connections. It is used as + * follows: + * + * class Connection { ... }; + * + * Pool pool; + * + * { + * auto conn(pool.get()); + * conn->exec("select ..."); + * } + * + * Here, the Connection object referenced by ‘conn’ is automatically + * returned to the pool when ‘conn’ goes out of scope. + */ template class Pool { public: - /* A function that produces new instances of R on demand. */ + /** + * A function that produces new instances of R on demand. + */ typedef std::function()> Factory; - /* A function that checks whether an instance of R is still - usable. Unusable instances are removed from the pool. */ + /** + * A function that checks whether an instance of R is still + * usable. Unusable instances are removed from the pool. + */ typedef std::function &)> Validator; private: diff -Nru nix-2.8.0/src/libutil/ref.hh nix-2.16.1+dfsg/src/libutil/ref.hh --- nix-2.8.0/src/libutil/ref.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/ref.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -6,8 +7,10 @@ namespace nix { -/* A simple non-nullable reference-counted pointer. Actually a wrapper - around std::shared_ptr that prevents non-null constructions. */ +/** + * A simple non-nullable reference-counted pointer. Actually a wrapper + * around std::shared_ptr that prevents null constructions. + */ template class ref { @@ -83,6 +86,11 @@ return p != other.p; } + bool operator < (const ref & other) const + { + return p < other.p; + } + private: template @@ -99,47 +107,4 @@ return ref(p); } - -/* A non-nullable pointer. - This is similar to a C++ "& reference", but mutable. - This is similar to ref but backed by a regular pointer instead of a smart pointer. - */ -template -class ptr { -private: - T * p; - -public: - ptr(const ptr & r) - : p(r.p) - { } - - explicit ptr(T * p) - : p(p) - { - if (!p) - throw std::invalid_argument("null pointer cast to ptr"); - } - - T* operator ->() const - { - return &*p; - } - - T& operator *() const - { - return *p; - } - - bool operator == (const ptr & other) const - { - return p == other.p; - } - - bool operator != (const ptr & other) const - { - return p != other.p; - } -}; - } diff -Nru nix-2.8.0/src/libutil/regex-combinators.hh nix-2.16.1+dfsg/src/libutil/regex-combinators.hh --- nix-2.8.0/src/libutil/regex-combinators.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/regex-combinators.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include + +namespace nix::regex { + +// TODO use constexpr string building like +// https://github.com/akrzemi1/static_string/blob/master/include/ak_toolkit/static_string.hpp + +static inline std::string either(std::string_view a, std::string_view b) +{ + return std::string { a } + "|" + b; +} + +static inline std::string group(std::string_view a) +{ + return std::string { "(" } + a + ")"; +} + +static inline std::string many(std::string_view a) +{ + return std::string { "(?:" } + a + ")*"; +} + +static inline std::string list(std::string_view a) +{ + return std::string { a } + many(group("," + a)); +} + +} diff -Nru nix-2.8.0/src/libutil/serialise.cc nix-2.16.1+dfsg/src/libutil/serialise.cc --- nix-2.8.0/src/libutil/serialise.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/serialise.cc 2023-06-06 10:55:23.000000000 +0000 @@ -20,7 +20,7 @@ buffer size. */ if (bufPos + data.size() >= bufSize) { flush(); - write(data); + writeUnbuffered(data); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer @@ -38,7 +38,7 @@ if (bufPos == 0) return; size_t n = bufPos; bufPos = 0; // don't trigger the assert() in ~BufferedSink() - write({buffer.get(), n}); + writeUnbuffered({buffer.get(), n}); } @@ -48,24 +48,9 @@ } -size_t threshold = 256 * 1024 * 1024; - -static void warnLargeDump() -{ - warn("dumping very large path (> 256 MiB); this may run out of memory"); -} - - -void FdSink::write(std::string_view data) +void FdSink::writeUnbuffered(std::string_view data) { written += data.size(); - static bool warned = false; - if (warn && !warned) { - if (written > threshold) { - warnLargeDump(); - warned = true; - } - } try { writeFull(fd, data); } catch (SysError & e) { @@ -201,6 +186,22 @@ StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; +std::shared_ptr (*create_coro_gc_hook)() = []() -> std::shared_ptr { + return {}; +}; + +/* This class is used for entry and exit hooks on coroutines */ +class CoroutineContext { + /* Disable GC when entering the coroutine without the boehm patch, + * since it doesn't find the main thread stack in this case. + * std::shared_ptr performs type-erasure, so it will call the right + * deleter. */ + const std::shared_ptr coro_gc_hook = create_coro_gc_hook(); +public: + CoroutineContext() {}; + ~CoroutineContext() {}; +}; + std::unique_ptr sourceToSink(std::function fun) { struct SourceToSink : FinishSink @@ -221,7 +222,8 @@ if (in.empty()) return; cur = in; - if (!coro) + if (!coro) { + CoroutineContext ctx; coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) { LambdaSource source([&](char *out, size_t out_len) { if (cur.empty()) { @@ -238,17 +240,24 @@ }); fun(source); }); + } if (!*coro) { abort(); } - if (!cur.empty()) (*coro)(false); + if (!cur.empty()) { + CoroutineContext ctx; + (*coro)(false); + } } void finish() override { if (!coro) return; if (!*coro) abort(); - (*coro)(true); + { + CoroutineContext ctx; + (*coro)(true); + } if (*coro) abort(); } }; @@ -279,18 +288,23 @@ size_t read(char * data, size_t len) override { - if (!coro) + if (!coro) { + CoroutineContext ctx; coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { LambdaSink sink([&](std::string_view data) { if (!data.empty()) yield(std::string(data)); }); fun(sink); }); + } if (!*coro) { eof(); abort(); } if (pos == cur.size()) { - if (!cur.empty()) (*coro)(); + if (!cur.empty()) { + CoroutineContext ctx; + (*coro)(); + } cur = coro->get(); pos = 0; } @@ -353,7 +367,7 @@ Sink & operator << (Sink & sink, const Error & ex) { - auto info = ex.info(); + auto & info = ex.info(); sink << "Error" << info.level @@ -430,7 +444,7 @@ auto msg = readString(source); ErrorInfo info { .level = level, - .msg = hintformat(std::move(format("%s") % msg)), + .msg = hintformat(fmt("%s", msg)), }; auto havePos = readNum(source); assert(havePos == 0); @@ -439,7 +453,7 @@ havePos = readNum(source); assert(havePos == 0); info.traces.push_back(Trace { - .hint = hintformat(std::move(format("%s") % readString(source))) + .hint = hintformat(fmt("%s", readString(source))) }); } return Error(std::move(info)); @@ -448,11 +462,6 @@ void StringSink::operator () (std::string_view data) { - static bool warned = false; - if (!warned && s.size() > threshold) { - warnLargeDump(); - warned = true; - } s.append(data); } diff -Nru nix-2.8.0/src/libutil/serialise.hh nix-2.16.1+dfsg/src/libutil/serialise.hh --- nix-2.8.0/src/libutil/serialise.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/serialise.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include @@ -10,7 +11,9 @@ namespace nix { -/* Abstract destination of binary data. */ +/** + * Abstract destination of binary data. + */ struct Sink { virtual ~Sink() { } @@ -18,7 +21,9 @@ virtual bool good() { return true; } }; -/* Just throws away data. */ +/** + * Just throws away data. + */ struct NullSink : Sink { void operator () (std::string_view data) override @@ -32,8 +37,10 @@ }; -/* A buffered abstract sink. Warning: a BufferedSink should not be - used from multiple threads concurrently. */ +/** + * A buffered abstract sink. Warning: a BufferedSink should not be + * used from multiple threads concurrently. + */ struct BufferedSink : virtual Sink { size_t bufSize, bufPos; @@ -46,23 +53,31 @@ void flush(); - virtual void write(std::string_view data) = 0; +protected: + + virtual void writeUnbuffered(std::string_view data) = 0; }; -/* Abstract source of binary data. */ +/** + * Abstract source of binary data. + */ struct Source { virtual ~Source() { } - /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. - It blocks until all the requested data is available, or throws - an error if it is not going to be available. */ + /** + * Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. + * It blocks until all the requested data is available, or throws + * an error if it is not going to be available. + */ void operator () (char * data, size_t len); - /* Store up to ‘len’ in the buffer pointed to by ‘data’, and - return the number of bytes stored. It blocks until at least - one byte is available. */ + /** + * Store up to ‘len’ in the buffer pointed to by ‘data’, and + * return the number of bytes stored. It blocks until at least + * one byte is available. + */ virtual size_t read(char * data, size_t len) = 0; virtual bool good() { return true; } @@ -73,8 +88,10 @@ }; -/* A buffered abstract source. Warning: a BufferedSource should not be - used from multiple threads concurrently. */ +/** + * A buffered abstract source. Warning: a BufferedSource should not be + * used from multiple threads concurrently. + */ struct BufferedSource : Source { size_t bufSize, bufPosIn, bufPosOut; @@ -88,35 +105,37 @@ bool hasData(); protected: - /* Underlying read call, to be overridden. */ + /** + * Underlying read call, to be overridden. + */ virtual size_t readUnbuffered(char * data, size_t len) = 0; }; -/* A sink that writes data to a file descriptor. */ +/** + * A sink that writes data to a file descriptor. + */ struct FdSink : BufferedSink { int fd; - bool warn = false; size_t written = 0; FdSink() : fd(-1) { } FdSink(int fd) : fd(fd) { } FdSink(FdSink&&) = default; - FdSink& operator=(FdSink && s) + FdSink & operator=(FdSink && s) { flush(); fd = s.fd; s.fd = -1; - warn = s.warn; written = s.written; return *this; } ~FdSink(); - void write(std::string_view data) override; + void writeUnbuffered(std::string_view data) override; bool good() override; @@ -125,7 +144,9 @@ }; -/* A source that reads data from a file descriptor. */ +/** + * A source that reads data from a file descriptor. + */ struct FdSource : BufferedSource { int fd; @@ -151,7 +172,9 @@ }; -/* A sink that writes data to a string. */ +/** + * A sink that writes data to a string. + */ struct StringSink : Sink { std::string s; @@ -165,7 +188,9 @@ }; -/* A source that reads data from a string. */ +/** + * A source that reads data from a string. + */ struct StringSource : Source { std::string_view s; @@ -175,7 +200,9 @@ }; -/* A sink that writes all incoming data to two other sinks. */ +/** + * A sink that writes all incoming data to two other sinks. + */ struct TeeSink : Sink { Sink & sink1, & sink2; @@ -188,7 +215,9 @@ }; -/* Adapter class of a Source that saves all data read to a sink. */ +/** + * Adapter class of a Source that saves all data read to a sink. + */ struct TeeSource : Source { Source & orig; @@ -203,7 +232,9 @@ } }; -/* A reader that consumes the original Source until 'size'. */ +/** + * A reader that consumes the original Source until 'size'. + */ struct SizedSource : Source { Source & orig; @@ -221,7 +252,9 @@ return n; } - /* Consume the original source until no remain data is left to consume. */ + /** + * Consume the original source until no remain data is left to consume. + */ size_t drainAll() { std::vector buf(8192); @@ -234,7 +267,9 @@ } }; -/* A sink that that just counts the number of bytes given to it */ +/** + * A sink that that just counts the number of bytes given to it + */ struct LengthSink : Sink { uint64_t length = 0; @@ -245,7 +280,9 @@ } }; -/* Convert a function into a sink. */ +/** + * Convert a function into a sink. + */ struct LambdaSink : Sink { typedef std::function lambda_t; @@ -261,7 +298,9 @@ }; -/* Convert a function into a source. */ +/** + * Convert a function into a source. + */ struct LambdaSource : Source { typedef std::function lambda_t; @@ -276,8 +315,10 @@ } }; -/* Chain two sources together so after the first is exhausted, the second is - used */ +/** + * Chain two sources together so after the first is exhausted, the second is + * used + */ struct ChainSource : Source { Source & source1, & source2; @@ -291,8 +332,10 @@ std::unique_ptr sourceToSink(std::function fun); -/* Convert a function that feeds data into a Sink into a Source. The - Source executes the function as a coroutine. */ +/** + * Convert a function that feeds data into a Sink into a Source. The + * Source executes the function as a coroutine. + */ std::unique_ptr sinkToSource( std::function fun, std::function eof = []() { @@ -333,17 +376,9 @@ unsigned char buf[8]; source((char *) buf, sizeof(buf)); - uint64_t n = - ((uint64_t) buf[0]) | - ((uint64_t) buf[1] << 8) | - ((uint64_t) buf[2] << 16) | - ((uint64_t) buf[3] << 24) | - ((uint64_t) buf[4] << 32) | - ((uint64_t) buf[5] << 40) | - ((uint64_t) buf[6] << 48) | - ((uint64_t) buf[7] << 56); + auto n = readLittleEndian(buf); - if (n > (uint64_t)std::numeric_limits::max()) + if (n > (uint64_t) std::numeric_limits::max()) throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); return (T) n; @@ -386,7 +421,9 @@ Error readError(Source & source); -/* An adapter that converts a std::basic_istream into a source. */ +/** + * An adapter that converts a std::basic_istream into a source. + */ struct StreamToSourceAdapter : Source { std::shared_ptr> istream; @@ -409,13 +446,14 @@ }; -/* A source that reads a distinct format of concatenated chunks back into its - logical form, in order to guarantee a known state to the original stream, - even in the event of errors. - - Use with FramedSink, which also allows the logical stream to be terminated - in the event of an exception. -*/ +/** + * A source that reads a distinct format of concatenated chunks back into its + * logical form, in order to guarantee a known state to the original stream, + * even in the event of errors. + * + * Use with FramedSink, which also allows the logical stream to be terminated + * in the event of an exception. + */ struct FramedSource : Source { Source & from; @@ -460,11 +498,12 @@ } }; -/* Write as chunks in the format expected by FramedSource. - - The exception_ptr reference can be used to terminate the stream when you - detect that an error has occurred on the remote end. -*/ +/** + * Write as chunks in the format expected by FramedSource. + * + * The exception_ptr reference can be used to terminate the stream when you + * detect that an error has occurred on the remote end. + */ struct FramedSink : nix::BufferedSink { BufferedSink & to; @@ -483,7 +522,7 @@ } } - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { /* Don't send more data if the remote has encountered an error. */ @@ -497,18 +536,27 @@ }; }; -/* Stack allocation strategy for sinkToSource. - Mutable to avoid a boehm gc dependency in libutil. - - boost::context doesn't provide a virtual class, so we define our own. +/** + * Stack allocation strategy for sinkToSource. + * Mutable to avoid a boehm gc dependency in libutil. + * + * boost::context doesn't provide a virtual class, so we define our own. */ struct StackAllocator { virtual boost::context::stack_context allocate() = 0; virtual void deallocate(boost::context::stack_context sctx) = 0; - /* The stack allocator to use in sinkToSource and potentially elsewhere. - It is reassigned by the initGC() method in libexpr. */ + /** + * The stack allocator to use in sinkToSource and potentially elsewhere. + * It is reassigned by the initGC() method in libexpr. + */ static StackAllocator *defaultAllocator; }; +/* Disabling GC when entering a coroutine (without the boehm patch). + mutable to avoid boehm gc dependency in libutil. + */ +extern std::shared_ptr (*create_coro_gc_hook)(); + + } diff -Nru nix-2.8.0/src/libutil/split.hh nix-2.16.1+dfsg/src/libutil/split.hh --- nix-2.8.0/src/libutil/split.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/split.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,10 +8,12 @@ namespace nix { -// If `separator` is found, we return the portion of the string before the -// separator, and modify the string argument to contain only the part after the -// separator. Otherwise, we return `std::nullopt`, and we leave the argument -// string alone. +/** + * If `separator` is found, we return the portion of the string before the + * separator, and modify the string argument to contain only the part after the + * separator. Otherwise, we return `std::nullopt`, and we leave the argument + * string alone. + */ static inline std::optional splitPrefixTo(std::string_view & string, char separator) { auto sepInstance = string.find(separator); diff -Nru nix-2.8.0/src/libutil/suggestions.hh nix-2.16.1+dfsg/src/libutil/suggestions.hh --- nix-2.8.0/src/libutil/suggestions.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/suggestions.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "comparator.hh" #include "types.hh" @@ -13,7 +14,8 @@ */ class Suggestion { public: - int distance; // The smaller the better + /// The smaller the better + int distance; std::string suggestion; std::string to_string() const; @@ -43,7 +45,9 @@ std::ostream & operator<<(std::ostream & str, const Suggestion &); std::ostream & operator<<(std::ostream & str, const Suggestions &); -// Either a value of type `T`, or some suggestions +/** + * Either a value of type `T`, or some suggestions + */ template class OrSuggestions { public: diff -Nru nix-2.8.0/src/libutil/sync.hh nix-2.16.1+dfsg/src/libutil/sync.hh --- nix-2.8.0/src/libutil/sync.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/sync.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,22 +8,22 @@ namespace nix { -/* This template class ensures synchronized access to a value of type - T. It is used as follows: - - struct Data { int x; ... }; - - Sync data; - - { - auto data_(data.lock()); - data_->x = 123; - } - - Here, "data" is automatically unlocked when "data_" goes out of - scope. -*/ - +/** + * This template class ensures synchronized access to a value of type + * T. It is used as follows: + * + * struct Data { int x; ... }; + * + * Sync data; + * + * { + * auto data_(data.lock()); + * data_->x = 123; + * } + * + * Here, "data" is automatically unlocked when "data_" goes out of + * scope. + */ template class Sync { diff -Nru nix-2.8.0/src/libutil/tarfile.cc nix-2.16.1+dfsg/src/libutil/tarfile.cc --- nix-2.8.0/src/libutil/tarfile.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tarfile.cc 2023-06-06 10:55:23.000000000 +0000 @@ -17,7 +17,7 @@ *buffer = self->buffer.data(); try { - return self->source->read((char *) self->buffer.data(), 4096); + return self->source->read((char *) self->buffer.data(), self->buffer.size()); } catch (EndOfFile &) { return 0; } catch (std::exception & err) { @@ -39,7 +39,7 @@ throw Error(reason, archive_error_string(this->archive)); } -TarArchive::TarArchive(Source & source, bool raw) : buffer(4096) +TarArchive::TarArchive(Source & source, bool raw) : buffer(65536) { this->archive = archive_read_new(); this->source = &source; @@ -77,9 +77,7 @@ static void extract_archive(TarArchive & archive, const Path & destDir) { - int flags = ARCHIVE_EXTRACT_FFLAGS - | ARCHIVE_EXTRACT_PERM - | ARCHIVE_EXTRACT_TIME + int flags = ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_SECURE_SYMLINKS | ARCHIVE_EXTRACT_SECURE_NODOTDOT; @@ -98,6 +96,10 @@ archive_entry_copy_pathname(entry, (destDir + "/" + name).c_str()); + // sources can and do contain dirs with no rx bits + if (archive_entry_filetype(entry) == AE_IFDIR && (archive_entry_mode(entry) & 0500) != 0500) + archive_entry_set_mode(entry, archive_entry_mode(entry) | 0500); + // Patch hardlink path const char *original_hardlink = archive_entry_hardlink(entry); if (original_hardlink) { diff -Nru nix-2.8.0/src/libutil/tarfile.hh nix-2.16.1+dfsg/src/libutil/tarfile.hh --- nix-2.8.0/src/libutil/tarfile.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tarfile.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "serialise.hh" #include @@ -14,13 +17,14 @@ TarArchive(const Path & path); - // disable copy constructor + /// disable copy constructor TarArchive(const TarArchive &) = delete; void close(); ~TarArchive(); }; + void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); diff -Nru nix-2.8.0/src/libutil/tests/canon-path.cc nix-2.16.1+dfsg/src/libutil/tests/canon-path.cc --- nix-2.8.0/src/libutil/tests/canon-path.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tests/canon-path.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,162 @@ +#include "canon-path.hh" + +#include + +namespace nix { + + TEST(CanonPath, basic) { + { + CanonPath p("/"); + ASSERT_EQ(p.abs(), "/"); + ASSERT_EQ(p.rel(), ""); + ASSERT_EQ(p.baseName(), std::nullopt); + ASSERT_EQ(p.dirOf(), std::nullopt); + ASSERT_FALSE(p.parent()); + } + + { + CanonPath p("/foo//"); + ASSERT_EQ(p.abs(), "/foo"); + ASSERT_EQ(p.rel(), "foo"); + ASSERT_EQ(*p.baseName(), "foo"); + ASSERT_EQ(*p.dirOf(), ""); // FIXME: do we want this? + ASSERT_EQ(p.parent()->abs(), "/"); + } + + { + CanonPath p("foo/bar"); + ASSERT_EQ(p.abs(), "/foo/bar"); + ASSERT_EQ(p.rel(), "foo/bar"); + ASSERT_EQ(*p.baseName(), "bar"); + ASSERT_EQ(*p.dirOf(), "/foo"); + ASSERT_EQ(p.parent()->abs(), "/foo"); + } + + { + CanonPath p("foo//bar/"); + ASSERT_EQ(p.abs(), "/foo/bar"); + ASSERT_EQ(p.rel(), "foo/bar"); + ASSERT_EQ(*p.baseName(), "bar"); + ASSERT_EQ(*p.dirOf(), "/foo"); + } + } + + TEST(CanonPath, pop) { + CanonPath p("foo/bar/x"); + ASSERT_EQ(p.abs(), "/foo/bar/x"); + p.pop(); + ASSERT_EQ(p.abs(), "/foo/bar"); + p.pop(); + ASSERT_EQ(p.abs(), "/foo"); + p.pop(); + ASSERT_EQ(p.abs(), "/"); + } + + TEST(CanonPath, removePrefix) { + CanonPath p1("foo/bar"); + CanonPath p2("foo/bar/a/b/c"); + ASSERT_EQ(p2.removePrefix(p1).abs(), "/a/b/c"); + ASSERT_EQ(p1.removePrefix(p1).abs(), "/"); + ASSERT_EQ(p1.removePrefix(CanonPath("/")).abs(), "/foo/bar"); + } + + TEST(CanonPath, iter) { + { + CanonPath p("a//foo/bar//"); + std::vector ss; + for (auto & c : p) ss.push_back(c); + ASSERT_EQ(ss, std::vector({"a", "foo", "bar"})); + } + + { + CanonPath p("/"); + std::vector ss; + for (auto & c : p) ss.push_back(c); + ASSERT_EQ(ss, std::vector()); + } + } + + TEST(CanonPath, concat) { + { + CanonPath p1("a//foo/bar//"); + CanonPath p2("xyzzy/bla"); + ASSERT_EQ((p1 + p2).abs(), "/a/foo/bar/xyzzy/bla"); + } + + { + CanonPath p1("/"); + CanonPath p2("/a/b"); + ASSERT_EQ((p1 + p2).abs(), "/a/b"); + } + + { + CanonPath p1("/a/b"); + CanonPath p2("/"); + ASSERT_EQ((p1 + p2).abs(), "/a/b"); + } + + { + CanonPath p("/foo/bar"); + ASSERT_EQ((p + "x").abs(), "/foo/bar/x"); + } + + { + CanonPath p("/"); + ASSERT_EQ((p + "foo" + "bar").abs(), "/foo/bar"); + } + } + + TEST(CanonPath, within) { + ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo"))); + ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar"))); + ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/"))); + ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/"))); + } + + TEST(CanonPath, sort) { + ASSERT_FALSE(CanonPath("foo") < CanonPath("foo")); + ASSERT_TRUE (CanonPath("foo") < CanonPath("foo/bar")); + ASSERT_TRUE (CanonPath("foo/bar") < CanonPath("foo!")); + ASSERT_FALSE(CanonPath("foo!") < CanonPath("foo")); + ASSERT_TRUE (CanonPath("foo") < CanonPath("foo!")); + } + + TEST(CanonPath, allowed) { + std::set allowed { + CanonPath("foo/bar"), + CanonPath("foo!"), + CanonPath("xyzzy"), + CanonPath("a/b/c"), + }; + + ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("/").isAllowed(allowed)); + } + + TEST(CanonPath, makeRelative) { + CanonPath d("/foo/bar"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar")), "."); + ASSERT_EQ(d.makeRelative(CanonPath("/foo")), ".."); + ASSERT_EQ(d.makeRelative(CanonPath("/")), "../.."); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar/xyzzy")), "xyzzy"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar/xyzzy/bla")), "xyzzy/bla"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/xyzzy/bla")), "../xyzzy/bla"); + ASSERT_EQ(d.makeRelative(CanonPath("/xyzzy/bla")), "../../xyzzy/bla"); + } +} diff -Nru nix-2.8.0/src/libutil/tests/chunked-vector.cc nix-2.16.1+dfsg/src/libutil/tests/chunked-vector.cc --- nix-2.8.0/src/libutil/tests/chunked-vector.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tests/chunked-vector.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,54 @@ +#include "chunked-vector.hh" + +#include + +namespace nix { + TEST(ChunkedVector, InitEmpty) { + auto v = ChunkedVector(100); + ASSERT_EQ(v.size(), 0); + } + + TEST(ChunkedVector, GrowsCorrectly) { + auto v = ChunkedVector(100); + for (auto i = 1; i < 20; i++) { + v.add(i); + ASSERT_EQ(v.size(), i); + } + } + + TEST(ChunkedVector, AddAndGet) { + auto v = ChunkedVector(100); + for (auto i = 1; i < 20; i++) { + auto [i2, idx] = v.add(i); + auto & i3 = v[idx]; + ASSERT_EQ(i, i2); + ASSERT_EQ(&i2, &i3); + } + } + + TEST(ChunkedVector, ForEach) { + auto v = ChunkedVector(100); + for (auto i = 1; i < 20; i++) { + v.add(i); + } + int count = 0; + v.forEach([&count](int elt) { + count++; + }); + ASSERT_EQ(count, v.size()); + } + + TEST(ChunkedVector, OverflowOK) { + // Similar to the AddAndGet, but intentionnally use a small + // initial ChunkedVector to force it to overflow + auto v = ChunkedVector(2); + for (auto i = 1; i < 20; i++) { + auto [i2, idx] = v.add(i); + auto & i3 = v[idx]; + ASSERT_EQ(i, i2); + ASSERT_EQ(&i2, &i3); + } + } + +} + diff -Nru nix-2.8.0/src/libutil/tests/config.cc nix-2.16.1+dfsg/src/libutil/tests/config.cc --- nix-2.8.0/src/libutil/tests/config.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tests/config.cc 2023-06-06 10:55:23.000000000 +0000 @@ -82,6 +82,7 @@ TestSetting() : AbstractSetting("test", "test", {}) {} void set(const std::string & value, bool append) override {} std::string to_string() const override { return {}; } + bool isAppendable() override { return false; } }; Config config; @@ -90,6 +91,7 @@ ASSERT_FALSE(config.set("test", "value")); config.addSetting(&setting); ASSERT_TRUE(config.set("test", "value")); + ASSERT_FALSE(config.set("extra-test", "value")); } TEST(Config, withInitialValue) { @@ -156,12 +158,54 @@ } TEST(Config, toJSONOnNonEmptyConfig) { + using nlohmann::literals::operator "" _json; Config config; - std::map settings; - Setting setting{&config, "", "name-of-the-setting", "description"}; + Setting setting{ + &config, + "", + "name-of-the-setting", + "description", + }; + setting.assign("value"); + + ASSERT_EQ(config.toJSON(), + R"#({ + "name-of-the-setting": { + "aliases": [], + "defaultValue": "", + "description": "description\n", + "documentDefault": true, + "value": "value", + "experimentalFeature": null + } + })#"_json); + } + + TEST(Config, toJSONOnNonEmptyConfigWithExperimentalSetting) { + using nlohmann::literals::operator "" _json; + Config config; + Setting setting{ + &config, + "", + "name-of-the-setting", + "description", + {}, + true, + Xp::Flakes, + }; setting.assign("value"); - ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","documentDefault":true,"value":"value"}})#"); + ASSERT_EQ(config.toJSON(), + R"#({ + "name-of-the-setting": { + "aliases": [], + "defaultValue": "", + "description": "description\n", + "documentDefault": true, + "value": "value", + "experimentalFeature": "flakes" + } + })#"_json); } TEST(Config, setSettingAlias) { diff -Nru nix-2.8.0/src/libutil/tests/fmt.cc nix-2.16.1+dfsg/src/libutil/tests/fmt.cc --- nix-2.8.0/src/libutil/tests/fmt.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tests/fmt.cc 1970-01-01 00:00:00.000000000 +0000 @@ -1,68 +0,0 @@ -#include "fmt.hh" - -#include - -#include - -namespace nix { -/* ----------- tests for fmt.hh -------------------------------------------------*/ - - TEST(hiliteMatches, noHighlight) { - ASSERT_STREQ(hiliteMatches("Hello, world!", std::vector(), "(", ")").c_str(), "Hello, world!"); - } - - TEST(hiliteMatches, simpleHighlight) { - std::string str = "Hello, world!"; - std::regex re = std::regex("world"); - auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); - ASSERT_STREQ( - hiliteMatches(str, matches, "(", ")").c_str(), - "Hello, (world)!" - ); - } - - TEST(hiliteMatches, multipleMatches) { - std::string str = "Hello, world, world, world, world, world, world, Hello!"; - std::regex re = std::regex("world"); - auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); - ASSERT_STREQ( - hiliteMatches(str, matches, "(", ")").c_str(), - "Hello, (world), (world), (world), (world), (world), (world), Hello!" - ); - } - - TEST(hiliteMatches, overlappingMatches) { - std::string str = "world, Hello, world, Hello, world, Hello, world, Hello, world!"; - std::regex re = std::regex("Hello, world"); - std::regex re2 = std::regex("world, Hello"); - auto v = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); - for(auto it = std::sregex_iterator(str.begin(), str.end(), re2); it != std::sregex_iterator(); ++it) { - v.push_back(*it); - } - ASSERT_STREQ( - hiliteMatches(str, v, "(", ")").c_str(), - "(world, Hello, world, Hello, world, Hello, world, Hello, world)!" - ); - } - - TEST(hiliteMatches, complexOverlappingMatches) { - std::string str = "legacyPackages.x86_64-linux.git-crypt"; - std::vector regexes = { - std::regex("t-cry"), - std::regex("ux\\.git-cry"), - std::regex("git-c"), - std::regex("pt"), - }; - std::vector matches; - for(auto regex : regexes) - { - for(auto it = std::sregex_iterator(str.begin(), str.end(), regex); it != std::sregex_iterator(); ++it) { - matches.push_back(*it); - } - } - ASSERT_STREQ( - hiliteMatches(str, matches, "(", ")").c_str(), - "legacyPackages.x86_64-lin(ux.git-crypt)" - ); - } -} diff -Nru nix-2.8.0/src/libutil/tests/git.cc nix-2.16.1+dfsg/src/libutil/tests/git.cc --- nix-2.8.0/src/libutil/tests/git.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tests/git.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,33 @@ +#include "git.hh" +#include + +namespace nix { + + TEST(GitLsRemote, parseSymrefLineWithReference) { + auto line = "ref: refs/head/main HEAD"; + auto res = git::parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, "HEAD"); + } + + TEST(GitLsRemote, parseSymrefLineWithNoReference) { + auto line = "ref: refs/head/main"; + auto res = git::parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, std::nullopt); + } + + TEST(GitLsRemote, parseObjectRefLine) { + auto line = "abc123 refs/head/main"; + auto res = git::parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Object); + ASSERT_EQ(res->target, "abc123"); + ASSERT_EQ(res->reference, "refs/head/main"); + } +} + diff -Nru nix-2.8.0/src/libutil/tests/hash.cc nix-2.16.1+dfsg/src/libutil/tests/hash.cc --- nix-2.8.0/src/libutil/tests/hash.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tests/hash.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,5 +1,12 @@ -#include "hash.hh" +#include + +#include #include +#include + +#include + +#include "tests/hash.hh" namespace nix { @@ -73,3 +80,16 @@ "c7d329eeb6dd26545e96e55b874be909"); } } + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + Hash hash(htSHA1); + for (size_t i = 0; i < hash.hashSize; ++i) + hash.hash[i] = *gen::arbitrary(); + return gen::just(hash); +} + +} diff -Nru nix-2.8.0/src/libutil/tests/hash.hh nix-2.16.1+dfsg/src/libutil/tests/hash.hh --- nix-2.8.0/src/libutil/tests/hash.hh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tests/hash.hh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,16 @@ +#pragma once +///@file + +#include + +#include + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff -Nru nix-2.8.0/src/libutil/tests/hilite.cc nix-2.16.1+dfsg/src/libutil/tests/hilite.cc --- nix-2.8.0/src/libutil/tests/hilite.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tests/hilite.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,66 @@ +#include "hilite.hh" + +#include + +namespace nix { +/* ----------- tests for fmt.hh -------------------------------------------------*/ + + TEST(hiliteMatches, noHighlight) { + ASSERT_STREQ(hiliteMatches("Hello, world!", std::vector(), "(", ")").c_str(), "Hello, world!"); + } + + TEST(hiliteMatches, simpleHighlight) { + std::string str = "Hello, world!"; + std::regex re = std::regex("world"); + auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); + ASSERT_STREQ( + hiliteMatches(str, matches, "(", ")").c_str(), + "Hello, (world)!" + ); + } + + TEST(hiliteMatches, multipleMatches) { + std::string str = "Hello, world, world, world, world, world, world, Hello!"; + std::regex re = std::regex("world"); + auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); + ASSERT_STREQ( + hiliteMatches(str, matches, "(", ")").c_str(), + "Hello, (world), (world), (world), (world), (world), (world), Hello!" + ); + } + + TEST(hiliteMatches, overlappingMatches) { + std::string str = "world, Hello, world, Hello, world, Hello, world, Hello, world!"; + std::regex re = std::regex("Hello, world"); + std::regex re2 = std::regex("world, Hello"); + auto v = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); + for(auto it = std::sregex_iterator(str.begin(), str.end(), re2); it != std::sregex_iterator(); ++it) { + v.push_back(*it); + } + ASSERT_STREQ( + hiliteMatches(str, v, "(", ")").c_str(), + "(world, Hello, world, Hello, world, Hello, world, Hello, world)!" + ); + } + + TEST(hiliteMatches, complexOverlappingMatches) { + std::string str = "legacyPackages.x86_64-linux.git-crypt"; + std::vector regexes = { + std::regex("t-cry"), + std::regex("ux\\.git-cry"), + std::regex("git-c"), + std::regex("pt"), + }; + std::vector matches; + for(auto regex : regexes) + { + for(auto it = std::sregex_iterator(str.begin(), str.end(), regex); it != std::sregex_iterator(); ++it) { + matches.push_back(*it); + } + } + ASSERT_STREQ( + hiliteMatches(str, matches, "(", ")").c_str(), + "legacyPackages.x86_64-lin(ux.git-crypt)" + ); + } +} diff -Nru nix-2.8.0/src/libutil/tests/json.cc nix-2.16.1+dfsg/src/libutil/tests/json.cc --- nix-2.8.0/src/libutil/tests/json.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tests/json.cc 1970-01-01 00:00:00.000000000 +0000 @@ -1,193 +0,0 @@ -#include "json.hh" -#include -#include - -namespace nix { - - /* ---------------------------------------------------------------------------- - * toJSON - * --------------------------------------------------------------------------*/ - - TEST(toJSON, quotesCharPtr) { - const char* input = "test"; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "\"test\""); - } - - TEST(toJSON, quotesStdString) { - std::string input = "test"; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "\"test\""); - } - - TEST(toJSON, convertsNullptrtoNull) { - auto input = nullptr; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "null"); - } - - TEST(toJSON, convertsNullToNull) { - const char* input = 0; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "null"); - } - - - TEST(toJSON, convertsFloat) { - auto input = 1.024f; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "1.024"); - } - - TEST(toJSON, convertsDouble) { - const double input = 1.024; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "1.024"); - } - - TEST(toJSON, convertsBool) { - auto input = false; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "false"); - } - - TEST(toJSON, quotesTab) { - std::stringstream out; - toJSON(out, "\t"); - - ASSERT_EQ(out.str(), "\"\\t\""); - } - - TEST(toJSON, quotesNewline) { - std::stringstream out; - toJSON(out, "\n"); - - ASSERT_EQ(out.str(), "\"\\n\""); - } - - TEST(toJSON, quotesCreturn) { - std::stringstream out; - toJSON(out, "\r"); - - ASSERT_EQ(out.str(), "\"\\r\""); - } - - TEST(toJSON, quotesCreturnNewLine) { - std::stringstream out; - toJSON(out, "\r\n"); - - ASSERT_EQ(out.str(), "\"\\r\\n\""); - } - - TEST(toJSON, quotesDoublequotes) { - std::stringstream out; - toJSON(out, "\""); - - ASSERT_EQ(out.str(), "\"\\\"\""); - } - - TEST(toJSON, substringEscape) { - std::stringstream out; - const char *s = "foo\t"; - toJSON(out, s+3, s + strlen(s)); - - ASSERT_EQ(out.str(), "\"\\t\""); - } - - /* ---------------------------------------------------------------------------- - * JSONObject - * --------------------------------------------------------------------------*/ - - TEST(JSONObject, emptyObject) { - std::stringstream out; - { - JSONObject t(out); - } - ASSERT_EQ(out.str(), "{}"); - } - - TEST(JSONObject, objectWithList) { - std::stringstream out; - { - JSONObject t(out); - auto l = t.list("list"); - l.elem("element"); - } - ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); - } - - TEST(JSONObject, objectWithListIndent) { - std::stringstream out; - { - JSONObject t(out, true); - auto l = t.list("list"); - l.elem("element"); - } - ASSERT_EQ(out.str(), -R"#({ - "list": [ - "element" - ] -})#"); - } - - TEST(JSONObject, objectWithPlaceholderAndList) { - std::stringstream out; - { - JSONObject t(out); - auto l = t.placeholder("list"); - l.list().elem("element"); - } - - ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); - } - - TEST(JSONObject, objectWithPlaceholderAndObject) { - std::stringstream out; - { - JSONObject t(out); - auto l = t.placeholder("object"); - l.object().attr("key", "value"); - } - - ASSERT_EQ(out.str(), R"#({"object":{"key":"value"}})#"); - } - - /* ---------------------------------------------------------------------------- - * JSONList - * --------------------------------------------------------------------------*/ - - TEST(JSONList, empty) { - std::stringstream out; - { - JSONList l(out); - } - ASSERT_EQ(out.str(), R"#([])#"); - } - - TEST(JSONList, withElements) { - std::stringstream out; - { - JSONList l(out); - l.elem("one"); - l.object(); - l.placeholder().write("three"); - } - ASSERT_EQ(out.str(), R"#(["one",{},"three"])#"); - } -} - diff -Nru nix-2.8.0/src/libutil/tests/local.mk nix-2.16.1+dfsg/src/libutil/tests/local.mk --- nix-2.8.0/src/libutil/tests/local.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tests/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -2,14 +2,28 @@ programs += libutil-tests +libutil-tests-exe_NAME = libnixutil-tests + +libutil-tests-exe_DIR := $(d) + +libutil-tests-exe_INSTALL_DIR := + +libutil-tests-exe_LIBS = libutil-tests + +libutil-tests-exe_LDFLAGS := $(GTEST_LIBS) + +libraries += libutil-tests + +libutil-tests_NAME = libnixutil-tests + libutil-tests_DIR := $(d) libutil-tests_INSTALL_DIR := libutil-tests_SOURCES := $(wildcard $(d)/*.cc) -libutil-tests_CXXFLAGS += -I src/libutil -I src/libexpr +libutil-tests_CXXFLAGS += -I src/libutil libutil-tests_LIBS = libutil -libutil-tests_LDFLAGS := $(GTEST_LIBS) +libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff -Nru nix-2.8.0/src/libutil/tests/tests.cc nix-2.16.1+dfsg/src/libutil/tests/tests.cc --- nix-2.8.0/src/libutil/tests/tests.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tests/tests.cc 2023-06-06 10:55:23.000000000 +0000 @@ -202,7 +202,7 @@ } TEST(pathExists, bogusPathDoesNotExist) { - ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes")); + ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); } /* ---------------------------------------------------------------------------- @@ -312,6 +312,42 @@ } /* ---------------------------------------------------------------------------- + * getLine + * --------------------------------------------------------------------------*/ + + TEST(getLine, all) { + { + auto [line, rest] = getLine("foo\nbar\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\r\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\n"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine("foo"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine(""); + ASSERT_EQ(line, ""); + ASSERT_EQ(rest, ""); + } + } + + /* ---------------------------------------------------------------------------- * toLower * --------------------------------------------------------------------------*/ @@ -548,7 +584,7 @@ TEST(get, emptyContainer) { StringMap s = { }; - auto expected = std::nullopt; + auto expected = nullptr; ASSERT_EQ(get(s, "one"), expected); } @@ -559,7 +595,23 @@ s["two"] = "er"; auto expected = "yi"; - ASSERT_EQ(get(s, "one"), expected); + ASSERT_EQ(*get(s, "one"), expected); + } + + TEST(getOr, emptyContainer) { + StringMap s = { }; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "yi"), expected); + } + + TEST(getOr, getFromContainer) { + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "nope"), expected); } /* ---------------------------------------------------------------------------- diff -Nru nix-2.8.0/src/libutil/tests/url.cc nix-2.16.1+dfsg/src/libutil/tests/url.cc --- nix-2.8.0/src/libutil/tests/url.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/tests/url.cc 2023-06-06 10:55:23.000000000 +0000 @@ -99,6 +99,27 @@ ASSERT_EQ(parsed, expected); } + TEST(parseURL, parsesFilePlusHttpsUrl) { + auto s = "file+https://www.example.org/video.mp4"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "file+https://www.example.org/video.mp4", + .base = "https://www.example.org/video.mp4", + .scheme = "file+https", + .authority = "www.example.org", + .path = "/video.mp4", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, rejectsAuthorityInUrlsWithFileTransportation) { + auto s = "file://www.example.org/video.mp4"; + ASSERT_THROW(parseURL(s), Error); + } TEST(parseURL, parseIPv4Address) { auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello"; @@ -280,5 +301,38 @@ ASSERT_EQ(d, s); } + + + /* ---------------------------------------------------------------------------- + * percentEncode + * --------------------------------------------------------------------------*/ + + TEST(percentEncode, encodesUrlEncodedString) { + std::string s = percentEncode("==@=="); + std::string d = "%3D%3D%40%3D%3D"; + ASSERT_EQ(d, s); + } + + TEST(percentEncode, keepArgument) { + std::string a = percentEncode("abd / def"); + std::string b = percentEncode("abd / def", "/"); + ASSERT_EQ(a, "abd%20%2F%20def"); + ASSERT_EQ(b, "abd%20/%20def"); + } + + TEST(percentEncode, inverseOfDecode) { + std::string original = "%3D%3D%40%3D%3D"; + std::string once = percentEncode(original); + std::string back = percentDecode(once); + + ASSERT_EQ(back, original); + } + + TEST(percentEncode, trailingPercent) { + std::string s = percentEncode("==@==%"); + std::string d = "%3D%3D%40%3D%3D%25"; + + ASSERT_EQ(d, s); + } } diff -Nru nix-2.8.0/src/libutil/thread-pool.hh nix-2.16.1+dfsg/src/libutil/thread-pool.hh --- nix-2.8.0/src/libutil/thread-pool.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/thread-pool.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "sync.hh" #include "util.hh" @@ -13,8 +14,10 @@ MakeError(ThreadPoolShutDown, Error); -/* A simple thread pool that executes a queue of work items - (lambdas). */ +/** + * A simple thread pool that executes a queue of work items + * (lambdas). + */ class ThreadPool { public: @@ -23,19 +26,30 @@ ~ThreadPool(); - // FIXME: use std::packaged_task? + /** + * An individual work item. + * + * \todo use std::packaged_task? + */ typedef std::function work_t; - /* Enqueue a function to be executed by the thread pool. */ + /** + * Enqueue a function to be executed by the thread pool. + */ void enqueue(const work_t & t); - /* Execute work items until the queue is empty. Note that work - items are allowed to add new items to the queue; this is - handled correctly. Queue processing stops prematurely if any - work item throws an exception. This exception is propagated to - the calling thread. If multiple work items throw an exception - concurrently, only one item is propagated; the others are - printed on stderr and otherwise ignored. */ + /** + * Execute work items until the queue is empty. + * + * \note Note that work items are allowed to add new items to the + * queue; this is handled correctly. + * + * Queue processing stops prematurely if any work item throws an + * exception. This exception is propagated to the calling thread. If + * multiple work items throw an exception concurrently, only one + * item is propagated; the others are printed on stderr and + * otherwise ignored. + */ void process(); private: @@ -62,9 +76,11 @@ void shutdown(); }; -/* Process in parallel a set of items of type T that have a partial - ordering between them. Thus, any item is only processed after all - its dependencies have been processed. */ +/** + * Process in parallel a set of items of type T that have a partial + * ordering between them. Thus, any item is only processed after all + * its dependencies have been processed. + */ template void processGraph( ThreadPool & pool, diff -Nru nix-2.8.0/src/libutil/topo-sort.hh nix-2.16.1+dfsg/src/libutil/topo-sort.hh --- nix-2.8.0/src/libutil/topo-sort.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/topo-sort.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "error.hh" diff -Nru nix-2.8.0/src/libutil/types.hh nix-2.16.1+dfsg/src/libutil/types.hh --- nix-2.8.0/src/libutil/types.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/types.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,10 +1,12 @@ #pragma once +///@file #include "ref.hh" #include #include #include +#include #include #include #include @@ -16,7 +18,9 @@ typedef std::map StringMap; typedef std::map StringPairs; -/* Paths are just strings. */ +/** + * Paths are just strings. + */ typedef std::string Path; typedef std::string_view PathView; typedef std::list Paths; @@ -24,15 +28,19 @@ typedef std::vector> Headers; -/* Helper class to run code at startup. */ +/** + * Helper class to run code at startup. + */ template struct OnStartup { OnStartup(T && t) { t(); } }; -/* Wrap bools to prevent string literals (i.e. 'char *') from being - cast to a bool in Attr. */ +/** + * Wrap bools to prevent string literals (i.e. 'char *') from being + * cast to a bool in Attr. + */ template struct Explicit { T t; @@ -44,21 +52,25 @@ }; -/* This wants to be a little bit like rust's Cow type. - Some parts of the evaluator benefit greatly from being able to reuse - existing allocations for strings, but have to be able to also use - newly allocated storage for values. - - We do not define implicit conversions, even with ref qualifiers, - since those can easily become ambiguous to the reader and can degrade - into copying behaviour we want to avoid. */ +/** + * This wants to be a little bit like rust's Cow type. + * Some parts of the evaluator benefit greatly from being able to reuse + * existing allocations for strings, but have to be able to also use + * newly allocated storage for values. + * + * We do not define implicit conversions, even with ref qualifiers, + * since those can easily become ambiguous to the reader and can degrade + * into copying behaviour we want to avoid. + */ class BackedStringView { private: std::variant data; - /* Needed to introduce a temporary since operator-> must return - a pointer. Without this we'd need to store the view object - even when we already own a string. */ + /** + * Needed to introduce a temporary since operator-> must return + * a pointer. Without this we'd need to store the view object + * even when we already own a string. + */ class Ptr { private: std::string_view view; @@ -76,8 +88,10 @@ BackedStringView(const BackedStringView &) = delete; BackedStringView & operator=(const BackedStringView &) = delete; - /* We only want move operations defined since the sole purpose of - this type is to avoid copies. */ + /** + * We only want move operations defined since the sole purpose of + * this type is to avoid copies. + */ BackedStringView(BackedStringView && other) = default; BackedStringView & operator=(BackedStringView && other) = default; diff -Nru nix-2.8.0/src/libutil/url.cc nix-2.16.1+dfsg/src/libutil/url.cc --- nix-2.8.0/src/libutil/url.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/url.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,7 @@ #include "url.hh" #include "url-parts.hh" #include "util.hh" +#include "split.hh" namespace nix { @@ -29,13 +30,13 @@ auto & query = match[6]; auto & fragment = match[7]; - auto isFile = scheme.find("file") != std::string::npos; + auto transportIsFile = parseUrlScheme(scheme).transport == "file"; - if (authority && *authority != "" && isFile) + if (authority && *authority != "" && transportIsFile) throw BadURL("file:// URL '%s' has unexpected authority '%s'", url, *authority); - if (isFile && path.empty()) + if (transportIsFile && path.empty()) path = "/"; return ParsedURL{ @@ -87,17 +88,22 @@ return result; } -std::string percentEncode(std::string_view s) +const static std::string allowedInQuery = ":@/?"; +const static std::string allowedInPath = ":@/"; + +std::string percentEncode(std::string_view s, std::string_view keep) { std::string res; for (auto & c : s) + // unreserved + keep if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') - || strchr("-._~!$&'()*+,;=:@", c)) + || strchr("-._~", c) + || keep.find(c) != std::string::npos) res += c; else - res += fmt("%%%02x", (unsigned int) c); + res += fmt("%%%02X", (unsigned int) c); return res; } @@ -108,9 +114,9 @@ for (auto & [name, value] : ss) { if (!first) res += '&'; first = false; - res += percentEncode(name); + res += percentEncode(name, allowedInQuery); res += '='; - res += percentEncode(value); + res += percentEncode(value, allowedInQuery); } return res; } @@ -121,7 +127,7 @@ scheme + ":" + (authority ? "//" + *authority : "") - + path + + percentEncode(path, allowedInPath) + (query.empty() ? "" : "?" + encodeQuery(query)) + (fragment.empty() ? "" : "#" + percentEncode(fragment)); } @@ -136,4 +142,21 @@ && fragment == other.fragment; } +/** + * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme' + * into a tuple '(applicationScheme, transportScheme)' + * + * > parseUrlScheme("http") == ParsedUrlScheme{ {}, "http"} + * > parseUrlScheme("tarball+http") == ParsedUrlScheme{ {"tarball"}, "http"} + */ +ParsedUrlScheme parseUrlScheme(std::string_view scheme) +{ + auto application = splitPrefixTo(scheme, '+'); + auto transport = scheme; + return ParsedUrlScheme { + .application = application, + .transport = transport, + }; +} + } diff -Nru nix-2.8.0/src/libutil/url.hh nix-2.16.1+dfsg/src/libutil/url.hh --- nix-2.8.0/src/libutil/url.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/url.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "error.hh" @@ -7,7 +8,8 @@ struct ParsedURL { std::string url; - std::string base; // URL without query/fragment + /// URL without query/fragment + std::string base; std::string scheme; std::optional authority; std::string path; @@ -22,9 +24,25 @@ MakeError(BadURL, Error); std::string percentDecode(std::string_view in); +std::string percentEncode(std::string_view s, std::string_view keep=""); std::map decodeQuery(const std::string & query); ParsedURL parseURL(const std::string & url); +/** + * Although that’s not really standardized anywhere, an number of tools + * use a scheme of the form 'x+y' in urls, where y is the “transport layer” + * scheme, and x is the “application layer” scheme. + * + * For example git uses `git+https` to designate remotes using a Git + * protocol over http. + */ +struct ParsedUrlScheme { + std::optional application; + std::string_view transport; +}; + +ParsedUrlScheme parseUrlScheme(std::string_view scheme); + } diff -Nru nix-2.8.0/src/libutil/url-parts.hh nix-2.16.1+dfsg/src/libutil/url-parts.hh --- nix-2.8.0/src/libutil/url-parts.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/url-parts.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -22,21 +23,22 @@ const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; -// A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.\\/-]*"; // FIXME: check +/// A Git ref (i.e. branch or tag name). +/// \todo check that this is correct. +const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-]*"; extern std::regex refRegex; -// Instead of defining what a good Git Ref is, we define what a bad Git Ref is -// This is because of the definition of a ref in refs.c in https://github.com/git/git -// See tests/fetchGitRefs.sh for the full definition +/// Instead of defining what a good Git Ref is, we define what a bad Git Ref is +/// This is because of the definition of a ref in refs.c in https://github.com/git/git +/// See tests/fetchGitRefs.sh for the full definition const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; extern std::regex badGitRefRegex; -// A Git revision (a SHA-1 commit hash). +/// A Git revision (a SHA-1 commit hash). const static std::string revRegexS = "[0-9a-fA-F]{40}"; extern std::regex revRegex; -// A ref or revision, or a ref followed by a revision. +/// A ref or revision, or a ref followed by a revision. const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; diff -Nru nix-2.8.0/src/libutil/util.cc nix-2.16.1+dfsg/src/libutil/util.cc --- nix-2.8.0/src/libutil/util.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/util.cc 2023-06-06 10:55:23.000000000 +0000 @@ -2,6 +2,7 @@ #include "sync.hh" #include "finally.hh" #include "serialise.hh" +#include "cgroup.hh" #include #include @@ -29,11 +30,15 @@ #ifdef __APPLE__ #include +#include #endif #ifdef __linux__ #include #include +#include + +#include #endif @@ -42,6 +47,9 @@ namespace nix { +void initLibUtil() { +} + std::optional getEnv(const std::string & key) { char * value = getenv(key.c_str()); @@ -49,6 +57,11 @@ return std::string(value); } +std::optional getEnvNonEmpty(const std::string & key) { + auto value = getEnv(key); + if (value == "") return {}; + return value; +} std::map getEnv() { @@ -253,6 +266,17 @@ return false; } +bool pathAccessible(const Path & path) +{ + try { + return pathExists(path); + } catch (SysError & e) { + // swallow EPERM + if (e.errNo == EPERM) return false; + throw; + } +} + Path readLink(const Path & path) { @@ -349,7 +373,7 @@ } -void writeFile(const Path & path, std::string_view s, mode_t mode) +void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) @@ -360,10 +384,16 @@ e.addTrace({}, "writing file '%1%'", path); throw; } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); } -void writeFile(const Path & path, Source & source, mode_t mode) +void writeFile(const Path & path, Source & source, mode_t mode, bool sync) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) @@ -382,6 +412,20 @@ e.addTrace({}, "writing file '%1%'", path); throw; } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + +void syncParent(const Path & path) +{ + AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); } std::string readLine(int fd) @@ -498,67 +542,12 @@ void deletePath(const Path & path, uint64_t & bytesFreed) { - //Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path); + //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); bytesFreed = 0; _deletePath(path, bytesFreed); } -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - int & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); - else - return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); -} - - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static int globalCounter = 0; - int localCounter = 0; - int & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - - std::string getUserName() { auto pw = getpwuid(geteuid()); @@ -568,20 +557,41 @@ return name; } +Path getHomeOf(uid_t userId) +{ + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} Path getHome() { static Path homeDir = []() { + std::optional unownedUserHomeDir = {}; auto homeDir = getEnv("HOME"); + if (homeDir) { + // Only use $HOME if doesn't exist or is owned by the current user. + struct stat st; + int result = stat(homeDir->c_str(), &st); + if (result != 0) { + if (errno != ENOENT) { + warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + homeDir.reset(); + } + } else if (st.st_uid != geteuid()) { + unownedUserHomeDir.swap(homeDir); + } + } if (!homeDir) { - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - homeDir = pw->pw_dir; + homeDir = getHomeOf(geteuid()); + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } } return *homeDir; }(); @@ -618,6 +628,40 @@ return dataDir ? *dataDir : getHome() + "/.local/share"; } +Path getStateDir() +{ + auto stateDir = getEnv("XDG_STATE_HOME"); + return stateDir ? *stateDir : getHome() + "/.local/state"; +} + +Path createNixStateDir() +{ + Path dir = getStateDir() + "/nix"; + createDirs(dir); + return dir; +} + + +std::optional getSelfExe() +{ + static auto cached = []() -> std::optional + { + #if __linux__ + return readLink("/proc/self/exe"); + #elif __APPLE__ + char buf[1024]; + uint32_t size = sizeof(buf); + if (_NSGetExecutablePath(buf, &size) == 0) + return buf; + else + return std::nullopt; + #else + return std::nullopt; + #endif + }(); + return cached; +} + Paths createDirs(const Path & path) { @@ -642,44 +686,6 @@ } -void createSymlink(const Path & target, const Path & link, - std::optional mtime) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); - if (mtime) { - struct timeval times[2]; - times[0].tv_sec = *mtime; - times[0].tv_usec = 0; - times[1].tv_sec = *mtime; - times[1].tv_usec = 0; - if (lutimes(link.c_str(), times)) - throw SysError("setting time of symlink '%s'", link); - } -} - - -void replaceSymlink(const Path & target, const Path & link, - std::optional mtime) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp, mtime); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmp, link); - - break; - } -} - - void readFull(int fd, char * buf, size_t count) { while (count) { @@ -752,7 +758,32 @@ } } +////////////////////////////////////////////////////////////////////// + +unsigned int getMaxCPU() +{ + #if __linux__ + try { + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) return 0; + auto cgroups = getCgroups("/proc/self/cgroup"); + auto cgroup = cgroups[""]; + if (cgroup == "") return 0; + + auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; + + auto cpuMax = readFile(cpuFile); + auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + auto quota = cpuMaxParts[0]; + auto period = cpuMaxParts[1]; + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } catch (Error &) { ignoreException(lvlDebug); } + #endif + + return 0; +} ////////////////////////////////////////////////////////////////////// @@ -844,6 +875,20 @@ } } +void AutoCloseFD::fsync() +{ + if (fd != -1) { + int result; +#if __APPLE__ + result = ::fcntl(fd, F_FULLFSYNC); +#else + result = ::fsync(fd); +#endif + if (result == -1) + throw SysError("fsync file descriptor %1%", fd); + } +} + AutoCloseFD::operator bool() const { @@ -1039,9 +1084,19 @@ } +#if __linux__ +static int childEntry(void * arg) +{ + auto main = (std::function *) arg; + (*main)(); + return 1; +} +#endif + + pid_t startProcess(std::function fun, const ProcessOptions & options) { - auto wrapper = [&]() { + std::function wrapper = [&]() { if (!options.allowVfork) logger = makeSimpleLogger(); try { @@ -1061,7 +1116,27 @@ _exit(1); }; - pid_t pid = doFork(options.allowVfork, wrapper); + pid_t pid = -1; + + if (options.cloneFlags) { + #ifdef __linux__ + // Not supported, since then we don't know when to free the stack. + assert(!(options.cloneFlags & CLONE_VM)); + + size_t stackSize = 1 * 1024 * 1024; + auto stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + Finally freeStack([&]() { munmap(stack, stackSize); }); + + pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); + #else + throw Error("clone flags are only supported on Linux"); + #endif + } else + pid = doFork(options.allowVfork, wrapper); + if (pid == -1) throw SysError("unable to fork"); return pid; @@ -1077,12 +1152,12 @@ } std::string runProgram(Path program, bool searchPath, const Strings & args, - const std::optional & input) + const std::optional & input, bool isInteractive) { - auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input}); + auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); if (!statusOk(res.first)) - throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); return res.second; } @@ -1129,6 +1204,16 @@ // case), so we can't use it if we alter the environment processOptions.allowVfork = !options.environment; + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace( + []() { + logger->resume(); + } + ); + } + /* Fork. */ Pid pid = startProcess([&]() { if (options.environment) @@ -1210,7 +1295,7 @@ if (source) promise.get_future().get(); if (status) - throw ExecError(status, fmt("program '%1%' %2%", options.program, statusToString(status))); + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); } @@ -1340,14 +1425,14 @@ { if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { if (WIFEXITED(status)) - return (format("failed with exit code %1%") % WEXITSTATUS(status)).str(); + return fmt("failed with exit code %1%", WEXITSTATUS(status)); else if (WIFSIGNALED(status)) { int sig = WTERMSIG(status); #if HAVE_STRSIGNAL const char * description = strsignal(sig); - return (format("failed due to signal %1% (%2%)") % sig % description).str(); + return fmt("failed due to signal %1% (%2%)", sig, description); #else - return (format("failed due to signal %1%") % sig).str(); + return fmt("failed due to signal %1%", sig); #endif } else @@ -1396,7 +1481,7 @@ } -void ignoreException() +void ignoreException(Verbosity lvl) { /* Make sure no exceptions leave this function. printError() also throws when remote is closed. */ @@ -1404,7 +1489,7 @@ try { throw; } catch (std::exception & e) { - printError("error (ignored): %1%", e.what()); + printMsg(lvl, "error (ignored): %1%", e.what()); } } catch (...) { } } @@ -1416,7 +1501,7 @@ && !getEnv("NO_COLOR").has_value(); } -std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width) +std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) { std::string t, e; size_t w = 0; @@ -1586,8 +1671,22 @@ } -////////////////////////////////////////////////////////////////////// +std::pair getLine(std::string_view s) +{ + auto newline = s.find('\n'); + if (newline == s.npos) { + return {s, ""}; + } else { + auto line = s.substr(0, newline); + if (!line.empty() && line[line.size() - 1] == '\r') + line = line.substr(0, line.size() - 1); + return {line, s.substr(newline + 1)}; + } +} + + +////////////////////////////////////////////////////////////////////// static Sync> windowSize{{0, 0}}; @@ -1669,14 +1768,40 @@ } static sigset_t savedSignalMask; +static bool savedSignalMaskIsSet = false; -void startSignalHandlerThread() +void setChildSignalMask(sigset_t * sigs) { - updateWindowSize(); + assert(sigs); // C style function, but think of sigs as a reference +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE + sigemptyset(&savedSignalMask); + // There's no "assign" or "copy" function, so we rely on (math) idempotence + // of the or operator: a or a = a. + sigorset(&savedSignalMask, sigs, sigs); +#else + // Without sigorset, our best bet is to assume that sigset_t is a type that + // can be assigned directly, such as is the case for a sigset_t defined as + // an integer type. + savedSignalMask = *sigs; +#endif + + savedSignalMaskIsSet = true; +} + +void saveSignalMask() { if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) throw SysError("querying signal mask"); + savedSignalMaskIsSet = true; +} + +void startSignalHandlerThread() +{ + updateWindowSize(); + + saveSignalMask(); + sigset_t set; sigemptyset(&set); sigaddset(&set, SIGINT); @@ -1692,6 +1817,20 @@ static void restoreSignals() { + // If startSignalHandlerThread wasn't called, that means we're not running + // in a proper libmain process, but a process that presumably manages its + // own signal handlers. Such a process should call either + // - initNix(), to be a proper libmain process + // - startSignalHandlerThread(), to resemble libmain regarding signal + // handling only + // - saveSignalMask(), for processes that define their own signal handling + // thread + // TODO: Warn about this? Have a default signal mask? The latter depends on + // whether we should generally inherit signal masks from the caller. + // I don't know what the larger unix ecosystem expects from us here. + if (!savedSignalMaskIsSet) + return; + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) throw SysError("restoring signals"); } @@ -1819,7 +1958,7 @@ if (chmod(path.c_str(), mode) == -1) throw SysError("changing permissions on '%1%'", path); - if (listen(fdSocket.get(), 5) == -1) + if (listen(fdSocket.get(), 100) == -1) throw SysError("cannot listen on socket '%1%'", path); return fdSocket; @@ -1893,7 +2032,7 @@ // FIXME: move to libstore/build -void commonChildInit(Pipe & logPipe) +void commonChildInit() { logger = makeSimpleLogger(); @@ -1907,10 +2046,6 @@ if (setsid() == -1) throw SysError("creating a new session"); - /* Dup the write side of the logger pipe into stderr. */ - if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) - throw SysError("cannot pipe standard error into log file"); - /* Dup stderr to stdout. */ if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) throw SysError("cannot dup stderr into stdout"); diff -Nru nix-2.8.0/src/libutil/util.hh nix-2.16.1+dfsg/src/libutil/util.hh --- nix-2.8.0/src/libutil/util.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/util.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "error.hh" @@ -31,74 +32,122 @@ struct Sink; struct Source; +void initLibUtil(); -/* The system for which Nix is compiled. */ +/** + * The system for which Nix is compiled. + */ extern const std::string nativeSystem; -/* Return an environment variable. */ +/** + * @return an environment variable. + */ std::optional getEnv(const std::string & key); -/* Get the entire environment. */ +/** + * @return a non empty environment variable. Returns nullopt if the env + * variable is set to "" + */ +std::optional getEnvNonEmpty(const std::string & key); + +/** + * Get the entire environment. + */ std::map getEnv(); -/* Clear the environment. */ +/** + * Clear the environment. + */ void clearEnv(); -/* Return an absolutized path, resolving paths relative to the - specified directory, or the current directory otherwise. The path - is also canonicalised. */ +/** + * @return An absolutized path, resolving paths relative to the + * specified directory, or the current directory otherwise. The path + * is also canonicalised. + */ Path absPath(Path path, std::optional dir = {}, bool resolveSymlinks = false); -/* Canonicalise a path by removing all `.' or `..' components and - double or trailing slashes. Optionally resolves all symlink - components such that each component of the resulting path is *not* - a symbolic link. */ +/** + * Canonicalise a path by removing all `.` or `..` components and + * double or trailing slashes. Optionally resolves all symlink + * components such that each component of the resulting path is *not* + * a symbolic link. + */ Path canonPath(PathView path, bool resolveSymlinks = false); -/* Return the directory part of the given canonical path, i.e., - everything before the final `/'. If the path is the root or an - immediate child thereof (e.g., `/foo'), this means `/' - is returned.*/ +/** + * @return The directory part of the given canonical path, i.e., + * everything before the final `/`. If the path is the root or an + * immediate child thereof (e.g., `/foo`), this means `/` + * is returned. + */ Path dirOf(const PathView path); -/* Return the base name of the given canonical path, i.e., everything - following the final `/' (trailing slashes are removed). */ +/** + * @return the base name of the given canonical path, i.e., everything + * following the final `/` (trailing slashes are removed). + */ std::string_view baseNameOf(std::string_view path); -/* Perform tilde expansion on a path. */ +/** + * Perform tilde expansion on a path. + */ std::string expandTilde(std::string_view path); -/* Check whether 'path' is a descendant of 'dir'. Both paths must be - canonicalized. */ +/** + * Check whether 'path' is a descendant of 'dir'. Both paths must be + * canonicalized. + */ bool isInDir(std::string_view path, std::string_view dir); -/* Check whether 'path' is equal to 'dir' or a descendant of - 'dir'. Both paths must be canonicalized. */ +/** + * Check whether 'path' is equal to 'dir' or a descendant of + * 'dir'. Both paths must be canonicalized. + */ bool isDirOrInDir(std::string_view path, std::string_view dir); -/* Get status of `path'. */ +/** + * Get status of `path`. + */ struct stat stat(const Path & path); struct stat lstat(const Path & path); -/* Return true iff the given path exists. */ +/** + * @return true iff the given path exists. + */ bool pathExists(const Path & path); -/* Read the contents (target) of a symbolic link. The result is not - in any way canonicalised. */ +/** + * A version of pathExists that returns false on a permission error. + * Useful for inferring default paths across directories that might not + * be readable. + * @return true iff the given path can be accessed and exists + */ +bool pathAccessible(const Path & path); + +/** + * Read the contents (target) of a symbolic link. The result is not + * in any way canonicalised. + */ Path readLink(const Path & path); bool isLink(const Path & path); -/* Read the contents of a directory. The entries `.' and `..' are - removed. */ +/** + * Read the contents of a directory. The entries `.` and `..` are + * removed. + */ struct DirEntry { std::string name; ino_t ino; - unsigned char type; // one of DT_* + /** + * one of DT_* + */ + unsigned char type; DirEntry(std::string name, ino_t ino, unsigned char type) : name(std::move(name)), ino(ino), type(type) { } }; @@ -109,77 +158,151 @@ unsigned char getFileType(const Path & path); -/* Read the contents of a file into a string. */ +/** + * Read the contents of a file into a string. + */ std::string readFile(int fd); std::string readFile(const Path & path); void readFile(const Path & path, Sink & sink); -/* Write a string to a file. */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666); - -void writeFile(const Path & path, Source & source, mode_t mode = 0666); - -/* Read a line from a file descriptor. */ +/** + * Write a string to a file. + */ +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); + +void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); + +/** + * Flush a file's parent directory to disk + */ +void syncParent(const Path & path); + +/** + * Read a line from a file descriptor. + */ std::string readLine(int fd); -/* Write a line to a file descriptor. */ +/** + * Write a line to a file descriptor. + */ void writeLine(int fd, std::string s); -/* Delete a path; i.e., in the case of a directory, it is deleted - recursively. It's not an error if the path does not exist. The - second variant returns the number of bytes and blocks freed. */ +/** + * Delete a path; i.e., in the case of a directory, it is deleted + * recursively. It's not an error if the path does not exist. The + * second variant returns the number of bytes and blocks freed. + */ void deletePath(const Path & path); void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); -/* Return $HOME or the user's home directory from /etc/passwd. */ +/** + * @return the given user's home directory from /etc/passwd. + */ +Path getHomeOf(uid_t userId); + +/** + * @return $HOME or the user's home directory from /etc/passwd. + */ Path getHome(); -/* Return $XDG_CACHE_HOME or $HOME/.cache. */ +/** + * @return $XDG_CACHE_HOME or $HOME/.cache. + */ Path getCacheDir(); -/* Return $XDG_CONFIG_HOME or $HOME/.config. */ +/** + * @return $XDG_CONFIG_HOME or $HOME/.config. + */ Path getConfigDir(); -/* Return the directories to search for user configuration files */ +/** + * @return the directories to search for user configuration files + */ std::vector getConfigDirs(); -/* Return $XDG_DATA_HOME or $HOME/.local/share. */ +/** + * @return $XDG_DATA_HOME or $HOME/.local/share. + */ Path getDataDir(); -/* Create a directory and all its parents, if necessary. Returns the - list of created directories, in order of creation. */ +/** + * @return the path of the current executable. + */ +std::optional getSelfExe(); + +/** + * @return $XDG_STATE_HOME or $HOME/.local/state. + */ +Path getStateDir(); + +/** + * Create the Nix state directory and return the path to it. + */ +Path createNixStateDir(); + +/** + * Create a directory and all its parents, if necessary. Returns the + * list of created directories, in order of creation. + */ Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) { +inline Paths createDirs(PathView path) +{ return createDirs(Path(path)); } -/* Create a symlink. */ +/** + * Create a symlink. + */ void createSymlink(const Path & target, const Path & link, std::optional mtime = {}); -/* Atomically create or replace a symlink. */ +/** + * Atomically create or replace a symlink. + */ void replaceSymlink(const Path & target, const Path & link, std::optional mtime = {}); +void renameFile(const Path & src, const Path & dst); -/* Wrappers arount read()/write() that read/write exactly the - requested number of bytes. */ +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + + +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ void readFull(int fd, char * buf, size_t count); void writeFull(int fd, std::string_view s, bool allowInterrupts = true); MakeError(EndOfFile, Error); -/* Read a file descriptor until EOF occurs. */ +/** + * Read a file descriptor until EOF occurs. + */ std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); - -/* Automatic cleanup of resources. */ +/** + * If cgroups are active, attempt to calculate the number of CPUs available. + * If cgroups are unavailable or if cpu.max is set to "max", return 0. + */ +unsigned int getMaxCPU(); + +/** + * Automatic cleanup of resources. + */ class AutoDelete @@ -213,14 +336,19 @@ explicit operator bool() const; int release(); void close(); + void fsync(); }; -/* Create a temporary directory. */ +/** + * Create a temporary directory. + */ Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); -/* Create a temporary file, returning a file handle and its path. */ +/** + * Create a temporary file, returning a file handle and its path. + */ std::pair createTempFile(const Path & prefix = "nix"); @@ -263,29 +391,39 @@ }; -/* Kill all processes running under the specified uid by sending them - a SIGKILL. */ +/** + * Kill all processes running under the specified uid by sending them + * a SIGKILL. + */ void killUser(uid_t uid); -/* Fork a process that runs the given function, and return the child - pid to the caller. */ +/** + * Fork a process that runs the given function, and return the child + * pid to the caller. + */ struct ProcessOptions { std::string errorPrefix = ""; bool dieWithParent = true; bool runExitHandlers = false; bool allowVfork = false; + /** + * use clone() with the specified flags (Linux only) + */ + int cloneFlags = 0; }; pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); -/* Run a program and return its stdout in a string (i.e., like the - shell backtick operator). */ +/** + * Run a program and return its stdout in a string (i.e., like the + * shell backtick operator). + */ std::string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), - const std::optional & input = {}); + const std::optional & input = {}, bool isInteractive = false); struct RunOptions { @@ -300,6 +438,7 @@ Source * standardIn = nullptr; Sink * standardOut = nullptr; bool mergeStderrToStdout = false; + bool isInteractive = false; }; std::pair runProgram(RunOptions && options); @@ -307,25 +446,37 @@ void runProgram2(const RunOptions & options); -/* Change the stack size. */ +/** + * Change the stack size. + */ void setStackSize(size_t stackSize); -/* Restore the original inherited Unix process context (such as signal - masks, stack size). */ +/** + * Restore the original inherited Unix process context (such as signal + * masks, stack size). + + * See startSignalHandlerThread(), saveSignalMask(). + */ void restoreProcessContext(bool restoreMounts = true); -/* Save the current mount namespace. Ignored if called more than - once. */ +/** + * Save the current mount namespace. Ignored if called more than + * once. + */ void saveMountNamespace(); -/* Restore the mount namespace saved by saveMountNamespace(). Ignored - if saveMountNamespace() was never called. */ +/** + * Restore the mount namespace saved by saveMountNamespace(). Ignored + * if saveMountNamespace() was never called. + */ void restoreMountNamespace(); -/* Cause this thread to not share any FS attributes with the main - thread, because this causes setns() in restoreMountNamespace() to - fail. */ +/** + * Cause this thread to not share any FS attributes with the main + * thread, because this causes setns() in restoreMountNamespace() to + * fail. + */ void unshareFilesystem(); @@ -340,16 +491,22 @@ { } }; -/* Convert a list of strings to a null-terminated vector of char - *'s. The result must not be accessed beyond the lifetime of the - list of strings. */ +/** + * Convert a list of strings to a null-terminated vector of `char + * *`s. The result must not be accessed beyond the lifetime of the + * list of strings. + */ std::vector stringsToCharPtrs(const Strings & ss); -/* Close all file descriptors except those listed in the given set. - Good practice in child processes. */ +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ void closeMostFDs(const std::set & exceptions); -/* Set the close-on-exec flag for the given file descriptor. */ +/** + * Set the close-on-exec flag for the given file descriptor. + */ void closeOnExec(int fd); @@ -375,12 +532,16 @@ MakeError(FormatError, Error); -/* String tokenizer. */ +/** + * String tokenizer. + */ template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); -/* Concatenate the given strings with a separator between the - elements. */ +/** + * Concatenate the given strings with a separator between the + * elements. + */ template std::string concatStringsSep(const std::string_view sep, const C & ss) { @@ -405,7 +566,9 @@ } -/* Add quotes around a collection of strings. */ +/** + * Add quotes around a collection of strings. + */ template Strings quoteStrings(const C & c) { Strings res; @@ -414,17 +577,23 @@ return res; } - -/* Remove trailing whitespace from a string. FIXME: return - std::string_view. */ +/** + * Remove trailing whitespace from a string. + * + * \todo return std::string_view. + */ std::string chomp(std::string_view s); -/* Remove whitespace from the start and end of a string. */ +/** + * Remove whitespace from the start and end of a string. + */ std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t"); -/* Replace all occurrences of a string inside another string. */ +/** + * Replace all occurrences of a string inside another string. + */ std::string replaceStrings( std::string s, std::string_view from, @@ -434,14 +603,18 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites); -/* Convert the exit status of a child as returned by wait() into an - error string. */ +/** + * Convert the exit status of a child as returned by wait() into an + * error string. + */ std::string statusToString(int status); bool statusOk(int status); -/* Parse a string into an integer. */ +/** + * Parse a string into an integer. + */ template std::optional string2Int(const std::string_view s) { @@ -454,8 +627,10 @@ } } -/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or - 'T' denoting a binary unit prefix. */ +/** + * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or + * 'T' denoting a binary unit prefix. + */ template N string2IntWithUnitPrefix(std::string_view s) { @@ -476,7 +651,9 @@ throw UsageError("'%s' is not an integer", s); } -/* Parse a string into a float. */ +/** + * Parse a string into a float. + */ template std::optional string2Float(const std::string_view s) { @@ -488,70 +665,136 @@ } -/* Return true iff `s' starts with `prefix'. */ +/** + * Convert a little-endian integer to host order. + */ +template +T readLittleEndian(unsigned char * p) +{ + T x = 0; + for (size_t i = 0; i < sizeof(x); ++i, ++p) { + x |= ((T) *p) << (i * 8); + } + return x; +} + + +/** + * @return true iff `s` starts with `prefix`. + */ bool hasPrefix(std::string_view s, std::string_view prefix); -/* Return true iff `s' ends in `suffix'. */ +/** + * @return true iff `s` ends in `suffix`. + */ bool hasSuffix(std::string_view s, std::string_view suffix); -/* Convert a string to lower case. */ +/** + * Convert a string to lower case. + */ std::string toLower(const std::string & s); -/* Escape a string as a shell word. */ +/** + * Escape a string as a shell word. + */ std::string shellEscape(const std::string_view s); -/* Exception handling in destructors: print an error message, then - ignore the exception. */ -void ignoreException(); +/** + * Exception handling in destructors: print an error message, then + * ignore the exception. + */ +void ignoreException(Verbosity lvl = lvlError); -/* Tree formatting. */ +/** + * Tree formatting. + */ constexpr char treeConn[] = "├───"; constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; -/* Determine whether ANSI escape sequences are appropriate for the - present output. */ +/** + * Determine whether ANSI escape sequences are appropriate for the + * present output. + */ bool shouldANSI(); -/* Truncate a string to 'width' printable characters. If 'filterAll' - is true, all ANSI escape sequences are filtered out. Otherwise, - some escape sequences (such as colour setting) are copied but not - included in the character count. Also, tabs are expanded to - spaces. */ -std::string filterANSIEscapes(const std::string & s, +/** + * Truncate a string to 'width' printable characters. If 'filterAll' + * is true, all ANSI escape sequences are filtered out. Otherwise, + * some escape sequences (such as colour setting) are copied but not + * included in the character count. Also, tabs are expanded to + * spaces. + */ +std::string filterANSIEscapes(std::string_view s, bool filterAll = false, unsigned int width = std::numeric_limits::max()); -/* Base64 encoding/decoding. */ +/** + * Base64 encoding/decoding. + */ std::string base64Encode(std::string_view s); std::string base64Decode(std::string_view s); -/* Remove common leading whitespace from the lines in the string - 's'. For example, if every line is indented by at least 3 spaces, - then we remove 3 spaces from the start of every line. */ +/** + * Remove common leading whitespace from the lines in the string + * 's'. For example, if every line is indented by at least 3 spaces, + * then we remove 3 spaces from the start of every line. + */ std::string stripIndentation(std::string_view s); -/* Get a value for the specified key from an associate container. */ +/** + * Get the prefix of 's' up to and excluding the next line break (LF + * optionally preceded by CR), and the remainder following the line + * break. + */ +std::pair getLine(std::string_view s); + + +/** + * Get a value for the specified key from an associate container. + */ template -std::optional get(const T & map, const typename T::key_type & key) +const typename T::mapped_type * get(const T & map, const typename T::key_type & key) { auto i = map.find(key); - if (i == map.end()) return {}; - return std::optional(i->second); + if (i == map.end()) return nullptr; + return &i->second; } +template +typename T::mapped_type * get(T & map, const typename T::key_type & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &i->second; +} -/* Remove and return the first item from a container. */ +/** + * Get a value for the specified key from an associate container, or a default value if the key isn't present. + */ +template +const typename T::mapped_type & getOr(T & map, + const typename T::key_type & key, + const typename T::mapped_type & defaultValue) +{ + auto i = map.find(key); + if (i == map.end()) return defaultValue; + return i->second; +} + +/** + * Remove and return the first item from a container. + */ template std::optional remove_begin(T & c) { @@ -563,7 +806,9 @@ } -/* Remove and return the first item from a container. */ +/** + * Remove and return the first item from a container. + */ template std::optional pop(T & c) { @@ -578,25 +823,48 @@ class Callback; -/* Start a thread that handles various signals. Also block those signals - on the current thread (and thus any threads created by it). */ +/** + * Start a thread that handles various signals. Also block those signals + * on the current thread (and thus any threads created by it). + * Saves the signal mask before changing the mask to block those signals. + * See saveSignalMask(). + */ void startSignalHandlerThread(); +/** + * Saves the signal mask, which is the signal mask that nix will restore + * before creating child processes. + * See setChildSignalMask() to set an arbitrary signal mask instead of the + * current mask. + */ +void saveSignalMask(); + +/** + * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't + * necessarily match the current thread's mask. + * See saveSignalMask() to set the saved mask to the current mask. + */ +void setChildSignalMask(sigset_t *sigs); + struct InterruptCallback { virtual ~InterruptCallback() { }; }; -/* Register a function that gets called on SIGINT (in a non-signal - context). */ +/** + * Register a function that gets called on SIGINT (in a non-signal + * context). + */ std::unique_ptr createInterruptCallback( std::function callback); void triggerInterrupt(); -/* A RAII class that causes the current thread to receive SIGUSR1 when - the signal handler thread receives SIGINT. That is, this allows - SIGINT to be multiplexed to multiple threads. */ +/** + * A RAII class that causes the current thread to receive SIGUSR1 when + * the signal handler thread receives SIGINT. That is, this allows + * SIGINT to be multiplexed to multiple threads. + */ struct ReceiveInterrupts { pthread_t target; @@ -610,8 +878,10 @@ -/* A RAII helper that increments a counter on construction and - decrements it on destruction. */ +/** + * A RAII helper that increments a counter on construction and + * decrements it on destruction. + */ template struct MaintainCount { @@ -622,33 +892,50 @@ }; -/* Return the number of rows and columns of the terminal. */ +/** + * @return the number of rows and columns of the terminal. + */ std::pair getWindowSize(); -/* Used in various places. */ +/** + * Used in various places. + */ typedef std::function PathFilter; extern PathFilter defaultPathFilter; -/* Common initialisation performed in child processes. */ -void commonChildInit(Pipe & logPipe); - -/* Create a Unix domain socket. */ +/** + * Common initialisation performed in child processes. + */ +void commonChildInit(); + +/** + * Create a Unix domain socket. + */ AutoCloseFD createUnixDomainSocket(); -/* Create a Unix domain socket in listen mode. */ +/** + * Create a Unix domain socket in listen mode. + */ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); -/* Bind a Unix domain socket to a path. */ +/** + * Bind a Unix domain socket to a path. + */ void bind(int fd, const std::string & path); -/* Connect to a Unix domain socket. */ +/** + * Connect to a Unix domain socket. + */ void connect(int fd, const std::string & path); -// A Rust/Python-like enumerate() iterator adapter. -// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. +/** + * A Rust/Python-like enumerate() iterator adapter. + * + * Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. + */ template ())), typename = decltype(std::end(std::declval()))> @@ -658,23 +945,25 @@ { size_t i; TIter iter; - bool operator != (const iterator & other) const { return iter != other.iter; } - void operator ++ () { ++i; ++iter; } - auto operator * () const { return std::tie(i, *iter); } + constexpr bool operator != (const iterator & other) const { return iter != other.iter; } + constexpr void operator ++ () { ++i; ++iter; } + constexpr auto operator * () const { return std::tie(i, *iter); } }; struct iterable_wrapper { T iterable; - auto begin() { return iterator{ 0, std::begin(iterable) }; } - auto end() { return iterator{ 0, std::end(iterable) }; } + constexpr auto begin() { return iterator{ 0, std::begin(iterable) }; } + constexpr auto end() { return iterator{ 0, std::end(iterable) }; } }; return iterable_wrapper{ std::forward(iterable) }; } -// C++17 std::visit boilerplate +/** + * C++17 std::visit boilerplate + */ template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; @@ -682,4 +971,30 @@ std::string showBytes(uint64_t bytes); +/** + * Provide an addition operator between strings and string_views + * inexplicably omitted from the standard library. + */ +inline std::string operator + (const std::string & s1, std::string_view s2) +{ + auto s = s1; + s.append(s2); + return s; +} + +inline std::string operator + (std::string && s, std::string_view s2) +{ + s.append(s2); + return std::move(s); +} + +inline std::string operator + (std::string_view s1, const char * s2) +{ + std::string s; + s.reserve(s1.size() + strlen(s2)); + s.append(s1); + s.append(s2); + return s; +} + } diff -Nru nix-2.8.0/src/libutil/xml-writer.hh nix-2.16.1+dfsg/src/libutil/xml-writer.hh --- nix-2.8.0/src/libutil/xml-writer.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/libutil/xml-writer.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include #include diff -Nru nix-2.8.0/src/nix/add-to-store.cc nix-2.16.1+dfsg/src/nix/add-to-store.cc --- nix-2.8.0/src/nix/add-to-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/add-to-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -42,14 +42,18 @@ } ValidPathInfo info { - store->makeFixedOutputPath(ingestionMethod, hash, *namePart), + *store, + std::move(*namePart), + FixedOutputInfo { + .hash = { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, + .references = {}, + }, narHash, }; info.narSize = sink.s.size(); - info.ca = std::optional { FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - } }; if (!dryRun) { auto source = StringSource(sink.s); diff -Nru nix-2.8.0/src/nix/app.cc nix-2.16.1+dfsg/src/nix/app.cc --- nix-2.8.0/src/nix/app.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/app.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,50 +1,30 @@ #include "installables.hh" +#include "installable-derived-path.hh" +#include "installable-value.hh" #include "store-api.hh" #include "eval-inline.hh" #include "eval-cache.hh" #include "names.hh" #include "command.hh" #include "derivations.hh" +#include "downstream-placeholder.hh" namespace nix { -struct InstallableDerivedPath : Installable -{ - ref store; - const DerivedPath derivedPath; - - InstallableDerivedPath(ref store, const DerivedPath & derivedPath) - : store(store) - , derivedPath(derivedPath) - { - } - - - std::string what() const override { return derivedPath.to_string(*store); } - - DerivedPaths toDerivedPaths() override - { - return {derivedPath}; - } - - std::optional getStorePath() override - { - return std::nullopt; - } -}; - /** * Return the rewrites that are needed to resolve a string whose context is * included in `dependencies`. */ -StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies) +StringPairs resolveRewrites( + Store & store, + const std::vector & dependencies) { StringPairs res; for (auto & dep : dependencies) - if (auto drvDep = std::get_if(&dep)) + if (auto drvDep = std::get_if(&dep.path)) for (auto & [ outputName, outputPath ] : drvDep->outputs) res.emplace( - downstreamPlaceholder(store, drvDep->drvPath, outputName), + DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(), store.printStorePath(outputPath) ); return res; @@ -53,29 +33,54 @@ /** * Resolve the given string assuming the given context. */ -std::string resolveString(Store & store, const std::string & toResolve, const BuiltPaths dependencies) +std::string resolveString( + Store & store, + const std::string & toResolve, + const std::vector & dependencies) { auto rewrites = resolveRewrites(store, dependencies); return rewriteStrings(toResolve, rewrites); } -UnresolvedApp Installable::toApp(EvalState & state) +UnresolvedApp InstallableValue::toApp(EvalState & state) { auto cursor = getCursor(state); auto attrPath = cursor->getAttrPath(); auto type = cursor->getAttr("type")->getString(); - std::string expected = !attrPath.empty() && attrPath[0] == "apps" ? "app" : "derivation"; + std::string expected = !attrPath.empty() && + (state.symbols[attrPath[0]] == "apps" || state.symbols[attrPath[0]] == "defaultApp") + ? "app" : "derivation"; if (type != expected) throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expected); if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); - std::vector context2; - for (auto & [path, name] : context) - context2.push_back({path, {name}}); + std::vector context2; + for (auto & c : context) { + context2.emplace_back(std::visit(overloaded { + [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath { + /* We want all outputs of the drv */ + return DerivedPath::Built { + .drvPath = d.drvPath, + .outputs = OutputsSpec::All {}, + }; + }, + [&](const NixStringContextElem::Built & b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }; + }, + [&](const NixStringContextElem::Opaque & o) -> DerivedPath { + return DerivedPath::Opaque { + .path = o.path, + }; + }, + }, c.raw())); + } return UnresolvedApp{App { .context = std::move(context2), @@ -89,7 +94,7 @@ auto outputName = cursor->getAttr(state.sOutputName)->getString(); auto name = cursor->getAttr(state.sName)->getString(); auto aPname = cursor->maybeGetAttr("pname"); - auto aMeta = cursor->maybeGetAttr("meta"); + auto aMeta = cursor->maybeGetAttr(state.sMeta); auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr; auto mainProgram = aMainProgram @@ -99,7 +104,10 @@ : DrvName(name).name; auto program = outPath + "/bin/" + mainProgram; return UnresolvedApp { App { - .context = { { drvPath, {outputName} } }, + .context = { DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::Names { outputName }, + } }, .program = program, }}; } @@ -113,11 +121,11 @@ { auto res = unresolved; - std::vector> installableContext; + Installables installableContext; for (auto & ctxElt : unresolved.context) installableContext.push_back( - std::make_shared(store, ctxElt.toDerivedPath())); + make_ref(store, DerivedPath { ctxElt })); auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext); res.program = resolveString(*store, unresolved.program, builtContext); diff -Nru nix-2.8.0/src/nix/build.cc nix-2.16.1+dfsg/src/nix/build.cc --- nix-2.8.0/src/nix/build.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/build.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,17 +1,74 @@ -#include "eval.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" +#include "progress-bar.hh" #include using namespace nix; +nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) +{ + auto res = nlohmann::json::array(); + for (auto & t : paths) { + std::visit([&res, store](const auto & t) { + res.push_back(t.toJSON(store)); + }, t.raw()); + } + return res; +} + +nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, ref store) +{ + auto res = nlohmann::json::array(); + for (auto & b : buildables) { + std::visit([&](const auto & t) { + auto j = t.toJSON(store); + if (b.result) { + if (b.result->startTime) + j["startTime"] = b.result->startTime; + if (b.result->stopTime) + j["stopTime"] = b.result->stopTime; + if (b.result->cpuUser) + j["cpuUser"] = ((double) b.result->cpuUser->count()) / 1000000; + if (b.result->cpuSystem) + j["cpuSystem"] = ((double) b.result->cpuSystem->count()) / 1000000; + } + res.push_back(j); + }, b.path.raw()); + } + return res; +} + +// TODO deduplicate with other code also setting such out links. +static void createOutLinks(const Path& outLink, const std::vector& buildables, LocalFSStore& store2) +{ + for (const auto & [_i, buildable] : enumerate(buildables)) { + auto i = _i; + std::visit(overloaded { + [&](const BuiltPath::Opaque & bo) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + store2.addPermRoot(bo.path, absPath(symlink)); + }, + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + if (output.first != "out") symlink += fmt("-%s", output.first); + store2.addPermRoot(output.second, absPath(symlink)); + } + }, + }, buildable.path.raw()); + } +} + struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile { Path outLink = "result"; + bool printOutputPaths = false; BuildMode buildMode = bmNormal; CmdBuild() @@ -32,6 +89,12 @@ }); addFlag({ + .longName = "print-out-paths", + .description = "Print the resulting output paths", + .handler = {&printOutputPaths, true}, + }); + + addFlag({ .longName = "rebuild", .description = "Rebuild an already built package and compare the result to the existing store paths.", .handler = {&buildMode, bmCheck}, @@ -50,50 +113,55 @@ ; } - void run(ref store) override + void run(ref store, Installables && installables) override { if (dryRun) { std::vector pathsToBuild; - for (auto & i : installables) { - auto b = i->toDerivedPaths(); - pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); - } + for (auto & i : installables) + for (auto & b : i->toDerivedPaths()) + pathsToBuild.push_back(b.path); + printMissing(store, pathsToBuild, lvlError); + if (json) logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + return; } auto buildables = Installable::build( getEvalStore(), store, Realise::Outputs, - installables, buildMode); + installables, + repair ? bmRepair : buildMode); - if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); + if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump()); if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) - for (const auto & [_i, buildable] : enumerate(buildables)) { - auto i = _i; - std::visit(overloaded { - [&](const BuiltPath::Opaque & bo) { - std::string symlink = outLink; - if (i) symlink += fmt("-%d", i); - store2->addPermRoot(bo.path, absPath(symlink)); - }, - [&](const BuiltPath::Built & bfd) { - for (auto & output : bfd.outputs) { - std::string symlink = outLink; - if (i) symlink += fmt("-%d", i); - if (output.first != "out") symlink += fmt("-%s", output.first); - store2->addPermRoot(output.second, absPath(symlink)); - } - }, - }, buildable.raw()); - } + createOutLinks(outLink, buildables, *store2); + + if (printOutputPaths) { + stopProgressBar(); + for (auto & buildable : buildables) { + std::visit(overloaded { + [&](const BuiltPath::Opaque & bo) { + logger->cout(store->printStorePath(bo.path)); + }, + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { + logger->cout(store->printStorePath(output.second)); + } + }, + }, buildable.path.raw()); + } + } - updateProfile(buildables); + BuiltPaths buildables2; + for (auto & b : buildables) + buildables2.push_back(b.path); + updateProfile(buildables2); } }; diff -Nru nix-2.8.0/src/nix/build.md nix-2.16.1+dfsg/src/nix/build.md --- nix-2.8.0/src/nix/build.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/build.md 2023-06-06 10:55:23.000000000 +0000 @@ -25,6 +25,13 @@ lrwxrwxrwx 1 … result-1 -> /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2 ``` +* Build GNU Hello and print the resulting store path. + + ```console + # nix build nixpkgs#hello --print-out-paths + /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + ``` + * Build a specific output: ```console @@ -37,7 +44,7 @@ `release.nix`: ```console - # nix build -f release.nix build.x86_64-linux + # nix build --file release.nix build.x86_64-linux ``` * Build a NixOS system configuration from a flake, and make a profile @@ -75,7 +82,7 @@ # Description -`nix build` builds the specified *installables*. Installables that +`nix build` builds the specified *installables*. [Installables](./nix.md#installables) that resolve to derivations are built (or substituted if possible). Store path installables are substituted. diff -Nru nix-2.8.0/src/nix/bundle.cc nix-2.16.1+dfsg/src/nix/bundle.cc --- nix-2.8.0/src/nix/bundle.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/bundle.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,13 +1,15 @@ -#include "command.hh" +#include "installable-flake.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "fs-accessor.hh" +#include "eval-inline.hh" using namespace nix; -struct CmdBundle : InstallableCommand +struct CmdBundle : InstallableValueCommand { std::string bundler = "github:NixOS/bundlers"; std::optional outLink; @@ -69,16 +71,16 @@ return res; } - void run(ref store) override + void run(ref store, ref installable) override { auto evalState = getEvalState(); auto val = installable->toValue(*evalState).first; - auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); + auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, - evalState, std::move(bundlerFlakeRef), bundlerName, + evalState, std::move(bundlerFlakeRef), bundlerName, extendedOutputsSpec, {"bundlers." + settings.thisSystem.get() + ".default", "defaultBundler." + settings.thisSystem.get() }, @@ -96,22 +98,29 @@ if (!attr1) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - PathSet context2; - auto drvPath = evalState->coerceToStorePath(*attr1->pos, *attr1->value, context2); + NixStringContext context2; + auto drvPath = evalState->coerceToStorePath(attr1->pos, *attr1->value, context2, ""); auto attr2 = vRes->attrs->get(evalState->sOutPath); if (!attr2) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - auto outPath = evalState->coerceToStorePath(*attr2->pos, *attr2->value, context2); + auto outPath = evalState->coerceToStorePath(attr2->pos, *attr2->value, context2, ""); - store->buildPaths({ DerivedPath::Built { drvPath } }); + store->buildPaths({ + DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All { }, + }, + }); auto outPathS = store->printStorePath(outPath); if (!outLink) { - auto &attr = vRes->attrs->need(evalState->sName); - outLink = evalState->forceStringNoCtx(*attr.value,*attr.pos); + auto * attr = vRes->attrs->get(evalState->sName); + if (!attr) + throw Error("attribute 'name' missing"); + outLink = evalState->forceStringNoCtx(*attr->value, attr->pos, ""); } // TODO: will crash if not a localFSStore? diff -Nru nix-2.8.0/src/nix/bundle.md nix-2.16.1+dfsg/src/nix/bundle.md --- nix-2.8.0/src/nix/bundle.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/bundle.md 2023-06-06 10:55:23.000000000 +0000 @@ -29,7 +29,7 @@ # Description -`nix bundle`, by default, packs the closure of the *installable* into a single +`nix bundle`, by default, packs the closure of the [*installable*](./nix.md#installables) into a single self-extracting executable. See the [`bundlers` homepage](https://github.com/NixOS/bundlers) for more details. @@ -44,7 +44,7 @@ * `bundlers..default` -If an attribute *name* is given, `nix run` tries the following flake +If an attribute *name* is given, `nix bundle` tries the following flake output attributes: * `bundlers..` diff -Nru nix-2.8.0/src/nix/cat.cc nix-2.16.1+dfsg/src/nix/cat.cc --- nix-2.8.0/src/nix/cat.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/cat.cc 2023-06-06 10:55:23.000000000 +0000 @@ -17,7 +17,7 @@ if (st.type != FSAccessor::Type::tRegular) throw Error("path '%1%' is not a regular file", path); - std::cout << accessor->readFile(path); + writeFull(STDOUT_FILENO, accessor->readFile(path)); } }; diff -Nru nix-2.8.0/src/nix/copy.cc nix-2.16.1+dfsg/src/nix/copy.cc --- nix-2.8.0/src/nix/copy.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/copy.cc 2023-06-06 10:55:23.000000000 +0000 @@ -10,8 +10,6 @@ SubstituteFlag substitute = NoSubstitute; - using BuiltPathsCommand::run; - CmdCopy() : BuiltPathsCommand(true) { diff -Nru nix-2.8.0/src/nix/copy.md nix-2.16.1+dfsg/src/nix/copy.md --- nix-2.8.0/src/nix/copy.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/copy.md 2023-06-06 10:55:23.000000000 +0000 @@ -15,7 +15,7 @@ SSH: ```console - # nix copy -s --to ssh://server /run/current-system + # nix copy --substitute-on-destination --to ssh://server /run/current-system ``` The `-s` flag causes the remote machine to try to substitute missing diff -Nru nix-2.8.0/src/nix/daemon.cc nix-2.16.1+dfsg/src/nix/daemon.cc --- nix-2.8.0/src/nix/daemon.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/daemon.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,3 +1,5 @@ +///@file + #include "command.hh" #include "shared.hh" #include "local-store.hh" @@ -34,6 +36,58 @@ using namespace nix; using namespace nix::daemon; +/** + * Settings related to authenticating clients for the Nix daemon. + * + * For pipes we have little good information about the client side, but + * for Unix domain sockets we do. So currently these options implemented + * mandatory access control based on user names and group names (looked + * up and translated to UID/GIDs in the CLI process that runs the code + * in this file). + * + * No code outside of this file knows about these settings (this is not + * exposed in a header); all authentication and authorization happens in + * `daemon.cc`. + */ +struct AuthorizationSettings : Config { + + Setting trustedUsers{ + this, {"root"}, "trusted-users", + R"( + A list of names of users (separated by whitespace) that have + additional rights when connecting to the Nix daemon, such as the + ability to specify additional binary caches, or to import unsigned + NARs. You can also specify groups by prefixing them with `@`; for + instance, `@wheel` means all users in the `wheel` group. The default + is `root`. + + > **Warning** + > + > Adding a user to `trusted-users` is essentially equivalent to + > giving that user root access to the system. For example, the user + > can set `sandbox-paths` and thereby obtain read access to + > directories that are otherwise inacessible to them. + )"}; + + /** + * Who we trust to use the daemon in safe ways + */ + Setting allowedUsers{ + this, {"*"}, "allowed-users", + R"( + A list of names of users (separated by whitespace) that are allowed + to connect to the Nix daemon. As with the `trusted-users` option, + you can specify groups by prefixing them with `@`. Also, you can + allow all users by specifying `*`. The default is `*`. + + Note that trusted users are always allowed to connect. + )"}; +}; + +AuthorizationSettings authorizationSettings; + +static GlobalConfig::Register rSettings(&authorizationSettings); + #ifndef __linux__ #define SPLICE_F_MOVE 0 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) @@ -75,8 +129,36 @@ throw SysError("setting SIGCHLD handler"); } +/** + * @return Is the given user a member of this group? + * + * @param user User specified by username. + * + * @param group Group the user might be a member of. + */ +static bool matchUser(std::string_view user, const struct group & gr) +{ + for (char * * mem = gr.gr_mem; *mem; mem++) + if (user == std::string_view(*mem)) return true; + return false; +} -bool matchUser(const std::string & user, const std::string & group, const Strings & users) + +/** + * Does the given user (specified by user name and primary group name) + * match the given user/group whitelist? + * + * If the list allows all users: Yes. + * + * If the username is in the set: Yes. + * + * If the groupname is in the set: Yes. + * + * If the user is in another group which is in the set: yes. + * + * Otherwise: No. + */ +static bool matchUser(const std::string & user, const std::string & group, const Strings & users) { if (find(users.begin(), users.end(), "*") != users.end()) return true; @@ -89,8 +171,7 @@ if (group == i.substr(1)) return true; struct group * gr = getgrnam(i.c_str() + 1); if (!gr) continue; - for (char * * mem = gr->gr_mem; *mem; mem++) - if (user == std::string(*mem)) return true; + if (matchUser(user, *gr)) return true; } return false; @@ -108,7 +189,9 @@ }; -// Get the identity of the caller, if possible. +/** + * Get the identity of the caller, if possible. + */ static PeerInfo getPeerInfo(int remote) { PeerInfo peer = { false, 0, false, 0, false, 0 }; @@ -142,6 +225,9 @@ #define SD_LISTEN_FDS_START 3 +/** + * Open a store without a path info cache. + */ static ref openUncachedStore() { Store::Params params; // FIXME: get params from somewhere @@ -150,8 +236,49 @@ return openStore(settings.storeUri, params); } +/** + * Authenticate a potential client + * + * @param peer Information about other end of the connection, the client which + * wants to communicate with us. + * + * @return A pair of a `TrustedFlag`, whether the potential client is trusted, + * and the name of the user (useful for printing messages). + * + * If the potential client is not allowed to talk to us, we throw an `Error`. + */ +static std::pair authPeer(const PeerInfo & peer) +{ + TrustedFlag trusted = NotTrusted; + + struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; + std::string user = pw ? pw->pw_name : std::to_string(peer.uid); + + struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; + std::string group = gr ? gr->gr_name : std::to_string(peer.gid); + + const Strings & trustedUsers = authorizationSettings.trustedUsers; + const Strings & allowedUsers = authorizationSettings.allowedUsers; + + if (matchUser(user, group, trustedUsers)) + trusted = Trusted; + + if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) + throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); + + return { trusted, std::move(user) }; +} + -static void daemonLoop() +/** + * Run a server. The loop opens a socket and accepts new connections from that + * socket. + * + * @param forceTrustClientOpt If present, force trusting or not trusted + * the client. Otherwise, decide based on the authentication settings + * and user credentials (from the unix domain socket). + */ +static void daemonLoop(std::optional forceTrustClientOpt) { if (chdir("/") == -1) throw SysError("cannot change current directory"); @@ -194,27 +321,22 @@ closeOnExec(remote.get()); - TrustedFlag trusted = NotTrusted; - PeerInfo peer = getPeerInfo(remote.get()); - - struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; - std::string user = pw ? pw->pw_name : std::to_string(peer.uid); - - struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; - std::string group = gr ? gr->gr_name : std::to_string(peer.gid); - - Strings trustedUsers = settings.trustedUsers; - Strings allowedUsers = settings.allowedUsers; - - if (matchUser(user, group, trustedUsers)) - trusted = Trusted; - - if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) - throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); - - printInfo(format((std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : "")) - % (peer.pidKnown ? std::to_string(peer.pid) : "") - % (peer.uidKnown ? user : "")); + PeerInfo peer { .pidKnown = false }; + TrustedFlag trusted; + std::string user; + + if (forceTrustClientOpt) + trusted = *forceTrustClientOpt; + else { + peer = getPeerInfo(remote.get()); + auto [_trusted, _user] = authPeer(peer); + trusted = _trusted; + user = _user; + }; + + printInfo((std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""), + peer.pidKnown ? std::to_string(peer.pid) : "", + peer.uidKnown ? user : ""); // Fork a child to handle the connection. ProcessOptions options; @@ -241,15 +363,7 @@ // Handle the connection. FdSource from(remote.get()); FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, NotRecursive, [&](Store & store) { -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); -#endif - store.createUser(user, peer.uid); - }); + processConnection(openUncachedStore(), from, to, trusted, NotRecursive); exit(0); }, options); @@ -257,7 +371,7 @@ } catch (Interrupted & e) { return; } catch (Error & error) { - ErrorInfo ei = error.info(); + auto ei = error.info(); // FIXME: add to trace? ei.msg = hintfmt("error processing connection: %1%", ei.msg.str()); logError(ei); @@ -265,53 +379,91 @@ } } -static void runDaemon(bool stdio) +/** + * Forward a standard IO connection to the given remote store. + * + * We just act as a middleman blindly ferry output between the standard + * input/output and the remote store connection, not processing anything. + * + * Loops until standard input disconnects, or an error is encountered. + */ +static void forwardStdioConnection(RemoteStore & store) { + auto conn = store.openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(from, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return; + } + } +} + +/** + * Process a client connecting to us via standard input/output + * + * Unlike `forwardStdioConnection()` we do process commands ourselves in + * this case, not delegating to another daemon. + * + * @param trustClient Whether to trust the client. Forwarded directly to + * `processConnection()`. + */ +static void processStdioConnection(ref store, TrustedFlag trustClient) +{ + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + processConnection(store, from, to, trustClient, NotRecursive); +} + +/** + * Entry point shared between the new CLI `nix daemon` and old CLI + * `nix-daemon`. + * + * @param forceTrustClientOpt See `daemonLoop()` and the parameter with + * the same name over there for details. + */ +static void runDaemon(bool stdio, std::optional forceTrustClientOpt) { if (stdio) { - if (auto store = openUncachedStore().dynamic_pointer_cast()) { - auto conn = store->openConnectionWrapper(); - int from = conn->from.fd; - int to = conn->to.fd; - - auto nfds = std::max(from, STDIN_FILENO) + 1; - while (true) { - fd_set fds; - FD_ZERO(&fds); - FD_SET(from, &fds); - FD_SET(STDIN_FILENO, &fds); - if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) - throw SysError("waiting for data from client or server"); - if (FD_ISSET(from, &fds)) { - auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from daemon socket to stdout"); - else if (res == 0) - throw EndOfFile("unexpected EOF from daemon socket"); - } - if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from stdin to daemon socket"); - else if (res == 0) - return; - } - } - } else { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limiting access to those is explicitly - not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); - } + auto store = openUncachedStore(); + + // If --force-untrusted is passed, we cannot forward the connection and + // must process it ourselves (before delegating to the next store) to + // force untrusting the client. + if (auto remoteStore = store.dynamic_pointer_cast(); remoteStore && (!forceTrustClientOpt || *forceTrustClientOpt != NotTrusted)) + forwardStdioConnection(*remoteStore); + else + // `Trusted` is passed in the auto (no override case) because we + // cannot see who is on the other side of a plain pipe. Limiting + // access to those is explicitly not `nix-daemon`'s responsibility. + processStdioConnection(store, forceTrustClientOpt.value_or(Trusted)); } else - daemonLoop(); + daemonLoop(forceTrustClientOpt); } static int main_nix_daemon(int argc, char * * argv) { { auto stdio = false; + std::optional isTrustedOpt = std::nullopt; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--daemon") @@ -322,11 +474,20 @@ printVersion("nix-daemon"); else if (*arg == "--stdio") stdio = true; - else return false; + else if (*arg == "--force-trusted") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = Trusted; + } else if (*arg == "--force-untrusted") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = NotTrusted; + } else if (*arg == "--default-trust") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = std::nullopt; + } else return false; return true; }); - runDaemon(stdio); + runDaemon(stdio, isTrustedOpt); return 0; } @@ -352,7 +513,7 @@ void run(ref store) override { - runDaemon(false); + runDaemon(false, std::nullopt); } }; diff -Nru nix-2.8.0/src/nix/daemon.md nix-2.16.1+dfsg/src/nix/daemon.md --- nix-2.8.0/src/nix/daemon.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/daemon.md 2023-06-06 10:55:23.000000000 +0000 @@ -11,7 +11,7 @@ # Description This command runs the Nix daemon, which is a required component in -multi-user Nix installations. It performs build actions and other +multi-user Nix installations. It runs build tasks and other operations on the Nix store on behalf of non-root users. Usually you don't run the daemon directly; instead it's managed by a service management framework such as `systemd`. diff -Nru nix-2.8.0/src/nix/derivation-add.cc nix-2.16.1+dfsg/src/nix/derivation-add.cc --- nix-2.8.0/src/nix/derivation-add.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/derivation-add.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,45 @@ +// FIXME: rename to 'nix plan add' or 'nix derivation add'? + +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" +#include "derivations.hh" +#include + +using namespace nix; +using json = nlohmann::json; + +struct CmdAddDerivation : MixDryRun, StoreCommand +{ + std::string description() override + { + return "Add a store derivation"; + } + + std::string doc() override + { + return + #include "derivation-add.md" + ; + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto json = nlohmann::json::parse(drainFD(STDIN_FILENO)); + + auto drv = Derivation::fromJSON(*store, json); + + auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun); + + drv.checkInvariants(*store, drvPath); + + writeDerivation(*store, drv, NoRepair, dryRun); + + logger->cout("%s", store->printStorePath(drvPath)); + } +}; + +static auto rCmdAddDerivation = registerCommand2({"derivation", "add"}); diff -Nru nix-2.8.0/src/nix/derivation-add.md nix-2.16.1+dfsg/src/nix/derivation-add.md --- nix-2.8.0/src/nix/derivation-add.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/derivation-add.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,18 @@ +R""( + +# Description + +This command reads from standard input a JSON representation of a +[store derivation] to which an [*installable*](./nix.md#installables) evaluates. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. + +[store derivation]: ../../glossary.md#gloss-store-derivation + +The JSON format is documented under the [`derivation show`] command. + +[`derivation show`]: ./nix3-derivation-show.md + +)"" diff -Nru nix-2.8.0/src/nix/derivation.cc nix-2.16.1+dfsg/src/nix/derivation.cc --- nix-2.8.0/src/nix/derivation.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/derivation.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,25 @@ +#include "command.hh" + +using namespace nix; + +struct CmdDerivation : virtual NixMultiCommand +{ + CmdDerivation() : MultiCommand(RegisterCommand::getCommandsFor({"derivation"})) + { } + + std::string description() override + { + return "Work with derivations, Nix's notion of a build plan."; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix derivation' requires a sub-command."); + command->second->run(); + } +}; + +static auto rCmdDerivation = registerCommand("derivation"); diff -Nru nix-2.8.0/src/nix/derivation-show.cc nix-2.16.1+dfsg/src/nix/derivation-show.cc --- nix-2.8.0/src/nix/derivation-show.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/derivation-show.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,64 @@ +// FIXME: integrate this with nix path-info? +// FIXME: rename to 'nix store derivation show' or 'nix debug derivation show'? + +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" +#include "derivations.hh" +#include + +using namespace nix; +using json = nlohmann::json; + +struct CmdShowDerivation : InstallablesCommand +{ + bool recursive = false; + + CmdShowDerivation() + { + addFlag({ + .longName = "recursive", + .shortName = 'r', + .description = "Include the dependencies of the specified derivations.", + .handler = {&recursive, true} + }); + } + + std::string description() override + { + return "show the contents of a store derivation"; + } + + std::string doc() override + { + return + #include "derivation-show.md" + ; + } + + Category category() override { return catUtility; } + + void run(ref store, Installables && installables) override + { + auto drvPaths = Installable::toDerivations(store, installables, true); + + if (recursive) { + StorePathSet closure; + store->computeFSClosure(drvPaths, closure); + drvPaths = std::move(closure); + } + + json jsonRoot = json::object(); + + for (auto & drvPath : drvPaths) { + if (!drvPath.isDerivation()) continue; + + jsonRoot[store->printStorePath(drvPath)] = + store->readDerivation(drvPath).toJSON(*store); + } + logger->cout(jsonRoot.dump(2)); + } +}; + +static auto rCmdShowDerivation = registerCommand2({"derivation", "show"}); diff -Nru nix-2.8.0/src/nix/derivation-show.md nix-2.16.1+dfsg/src/nix/derivation-show.md --- nix-2.8.0/src/nix/derivation-show.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/derivation-show.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,109 @@ +R""( + +# Examples + +* Show the [store derivation] that results from evaluating the Hello + package: + + [store derivation]: ../../glossary.md#gloss-store-derivation + + ```console + # nix derivation show nixpkgs#hello + { + "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { + … + } + } + ``` + +* Show the full derivation graph (if available) that produced your + NixOS system: + + ```console + # nix derivation show -r /run/current-system + ``` + +* Print all files fetched using `fetchurl` by Firefox's dependency + graph: + + ```console + # nix derivation show -r nixpkgs#firefox \ + | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ + | uniq | sort + ``` + + Note that `.outputs.out.hash` selects *fixed-output derivations* + (derivations that produce output with a specified content hash), + while `.env.urls` selects derivations with a `urls` attribute. + +# Description + +This command prints on standard output a JSON representation of the +[store derivation]s to which [*installables*](./nix.md#installables) evaluate. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. + +By default, this command only shows top-level derivations, but with +`--recursive`, it also shows their dependencies. + +The JSON output is a JSON object whose keys are the store paths of the +derivations, and whose values are a JSON object with the following +fields: + +* `name`: The name of the derivation. This is used when calculating the + store paths of the derivation's outputs. + +* `outputs`: Information about the output paths of the + derivation. This is a JSON object with one member per output, where + the key is the output name and the value is a JSON object with these + fields: + + * `path`: The output path. + * `hashAlgo`: For fixed-output derivations, the hashing algorithm + (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a + NAR hash rather than a flat file hash. + * `hash`: For fixed-output derivations, the expected content hash in + base-16. + + Example: + + ```json + "outputs": { + "out": { + "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", + "hashAlgo": "r:sha256", + "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + } + } + ``` + +* `inputSrcs`: A list of store paths on which this derivation depends. + +* `inputDrvs`: A JSON object specifying the derivations on which this + derivation depends, and what outputs of those derivations. For + example, + + ```json + "inputDrvs": { + "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + } + ``` + + specifies that this derivation depends on the `dev` output of + `curl`, and the `out` output of `unzip`. + +* `system`: The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + +* `builder`: The absolute path of the program to be executed to run + the build. Typically this is the `bash` shell + (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). + +* `args`: The command-line arguments passed to the `builder`. + +* `env`: The environment passed to the `builder`. + +)"" diff -Nru nix-2.8.0/src/nix/describe-stores.cc nix-2.16.1+dfsg/src/nix/describe-stores.cc --- nix-2.8.0/src/nix/describe-stores.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/describe-stores.cc 1970-01-01 00:00:00.000000000 +0000 @@ -1,44 +0,0 @@ -#include "command.hh" -#include "common-args.hh" -#include "shared.hh" -#include "store-api.hh" - -#include - -using namespace nix; - -struct CmdDescribeStores : Command, MixJSON -{ - std::string description() override - { - return "show registered store types and their available options"; - } - - Category category() override { return catUtility; } - - void run() override - { - auto res = nlohmann::json::object(); - for (auto & implem : *Implementations::registered) { - auto storeConfig = implem.getConfig(); - auto storeName = storeConfig->name(); - res[storeName] = storeConfig->toJSON(); - } - if (json) { - std::cout << res; - } else { - for (auto & [storeName, storeConfig] : res.items()) { - std::cout << "## " << storeName << std::endl << std::endl; - for (auto & [optionName, optionDesc] : storeConfig.items()) { - std::cout << "### " << optionName << std::endl << std::endl; - std::cout << optionDesc["description"].get() << std::endl; - std::cout << "default: " << optionDesc["defaultValue"] << std::endl <("describe-stores"); diff -Nru nix-2.8.0/src/nix/develop.cc nix-2.16.1+dfsg/src/nix/develop.cc --- nix-2.8.0/src/nix/develop.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/develop.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,9 +1,10 @@ #include "eval.hh" -#include "command.hh" +#include "installable-flake.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "derivations.hh" #include "progress-bar.hh" #include "run.hh" @@ -18,6 +19,9 @@ Setting bashPrompt{this, "", "bash-prompt", "The bash prompt (`PS1`) in `nix develop` shells."}; + Setting bashPromptPrefix{this, "", "bash-prompt-prefix", + "Prefix prepended to the `PS1` environment variable in `nix develop` shells."}; + Setting bashPromptSuffix{this, "", "bash-prompt-suffix", "Suffix appended to the `PS1` environment variable in `nix develop` shells."}; }; @@ -161,6 +165,14 @@ { return vars == other.vars && bashFunctions == other.bashFunctions; } + + std::string getSystem() const + { + if (auto v = get(vars, "system")) + return getString(*v); + else + return settings.thisSystem; + } }; const static std::string getEnvSh = @@ -189,12 +201,14 @@ drv.env.erase("allowedRequisites"); drv.env.erase("disallowedReferences"); drv.env.erase("disallowedRequisites"); + drv.env.erase("name"); /* Rehash and write the derivation. FIXME: would be nice to use 'buildDerivation', but that's privileged. */ drv.name += "-env"; + drv.env.emplace("name", drv.name); drv.inputSrcs.insert(std::move(getEnvShPath)); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { for (auto & output : drv.outputs) { output.second = DerivationOutput::Deferred {}, drv.env[output.first] = hashPlaceholder(output.first); @@ -219,7 +233,12 @@ auto shellDrvPath = writeDerivation(*evalStore, drv); /* Build the derivation. */ - store->buildPaths({DerivedPath::Built{shellDrvPath}}, bmNormal, evalStore); + store->buildPaths( + { DerivedPath::Built { + .drvPath = shellDrvPath, + .outputs = OutputsSpec::All { }, + }}, + bmNormal, evalStore); for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) { assert(optPath); @@ -243,6 +262,7 @@ "NIX_LOG_FD", "NIX_REMOTE", "PPID", + "SHELL", "SHELLOPTS", "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt "TEMP", @@ -273,15 +293,27 @@ const BuildEnvironment & buildEnvironment, const Path & outputsDir = absPath(".") + "/outputs") { + // A list of colon-separated environment variables that should be + // prepended to, rather than overwritten, in order to keep the shell usable. + // Please keep this list minimal in order to avoid impurities. + static const char * const savedVars[] = { + "PATH", // for commands + "XDG_DATA_DIRS", // for loadable completion + }; + std::ostringstream out; out << "unset shellHook\n"; - out << "nix_saved_PATH=\"$PATH\"\n"; + for (auto & var : savedVars) { + out << fmt("%s=${%s:-}\n", var, var); + out << fmt("nix_saved_%s=\"$%s\"\n", var, var); + } buildEnvironment.toBash(out, ignoreVars); - out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; + for (auto & var : savedVars) + out << fmt("%s=\"$%s${nix_saved_%s:+:$nix_saved_%s}\"\n", var, var, var, var); out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) @@ -342,7 +374,7 @@ return res; } - StorePath getShellOutPath(ref store) + StorePath getShellOutPath(ref store, ref installable) { auto path = installable->getStorePath(); if (path && hasSuffix(path->to_string(), "-env")) @@ -360,9 +392,10 @@ } } - std::pair getBuildEnvironment(ref store) + std::pair + getBuildEnvironment(ref store, ref installable) { - auto shellOutPath = getShellOutPath(store); + auto shellOutPath = getShellOutPath(store, installable); auto strPath = store->printStorePath(shellOutPath); @@ -448,9 +481,9 @@ ; } - void run(ref store) override + void run(ref store, ref installable) override { - auto [buildEnvironment, gcroot] = getBuildEnvironment(store); + auto [buildEnvironment, gcroot] = getBuildEnvironment(store, installable); auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); @@ -482,6 +515,9 @@ if (developSettings.bashPrompt != "") script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt.get())); + if (developSettings.bashPromptPrefix != "") + script += fmt("[ -n \"$PS1\" ] && PS1=%s\"$PS1\";\n", + shellEscape(developSettings.bashPromptPrefix.get())); if (developSettings.bashPromptSuffix != "") script += fmt("[ -n \"$PS1\" ] && PS1+=%s;\n", shellEscape(developSettings.bashPromptSuffix.get())); @@ -502,18 +538,34 @@ nixpkgsLockFlags.inputOverrides = {}; nixpkgsLockFlags.inputUpdates = {}; - auto bashInstallable = std::make_shared( + auto nixpkgs = defaultNixpkgsFlakeRef(); + if (auto * i = dynamic_cast(&*installable)) + nixpkgs = i->nixpkgsFlakeRef(); + + auto bashInstallable = make_ref( this, state, - installable->nixpkgsFlakeRef(), + std::move(nixpkgs), "bashInteractive", + DefaultOutputs(), Strings{}, Strings{"legacyPackages." + settings.thisSystem.get() + "."}, nixpkgsLockFlags); - shell = store->printStorePath( - Installable::toStorePath(getEvalStore(), store, Realise::Outputs, OperateOn::Output, bashInstallable)) - + "/bin/bash"; + bool found = false; + + for (auto & path : Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) { + auto s = store->printStorePath(path) + "/bin/bash"; + if (pathExists(s)) { + shell = s; + found = true; + break; + } + } + + if (!found) + throw Error("package 'nixpkgs#bashInteractive' does not provide a 'bin/bash'"); + } catch (Error &) { ignoreException(); } @@ -526,7 +578,7 @@ // Need to chdir since phases assume in flake directory if (phase) { // chdir if installable is a flake of type git+file or path - auto installableFlake = std::dynamic_pointer_cast(installable); + auto installableFlake = installable.dynamic_pointer_cast(); if (installableFlake) { auto sourcePath = installableFlake->getLockedFlake()->flake.resolvedRef.input.getSourcePath(); if (sourcePath) { @@ -537,7 +589,7 @@ } } - runProgramInStore(store, shell, args); + runProgramInStore(store, shell, args, buildEnvironment.getSystem()); } }; @@ -557,9 +609,9 @@ Category category() override { return catUtility; } - void run(ref store) override + void run(ref store, ref installable) override { - auto buildEnvironment = getBuildEnvironment(store).first; + auto buildEnvironment = getBuildEnvironment(store, installable).first; stopProgressBar(); diff -Nru nix-2.8.0/src/nix/develop.md nix-2.16.1+dfsg/src/nix/develop.md --- nix-2.8.0/src/nix/develop.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/develop.md 2023-06-06 10:55:23.000000000 +0000 @@ -66,11 +66,17 @@ `nixpkgs#glibc` in `~/my-glibc` and want to compile another package against it. +* Run a series of script commands: + + ```console + # nix develop --command bash --command "mkdir build && cmake .. && make" + ``` + # Description `nix develop` starts a `bash` shell that provides an interactive build environment nearly identical to what Nix would use to build -*installable*. Inside this shell, environment variables and shell +[*installable*](./nix.md#installables). Inside this shell, environment variables and shell functions are set up so that you can interactively and incrementally build your package. @@ -80,8 +86,8 @@ recorded into a profile using `--profile`. The prompt used by the `bash` shell can be customised by setting the -`bash-prompt` and `bash-prompt-suffix` settings in `nix.conf` or in -the flake's `nixConfig` attribute. +`bash-prompt`, `bash-prompt-prefix`, and `bash-prompt-suffix` settings in +`nix.conf` or in the flake's `nixConfig` attribute. # Flake output attributes diff -Nru nix-2.8.0/src/nix/diff-closures.cc nix-2.16.1+dfsg/src/nix/diff-closures.cc --- nix-2.8.0/src/nix/diff-closures.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/diff-closures.cc 2023-06-06 10:55:23.000000000 +0000 @@ -97,7 +97,7 @@ items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added))); if (showDelta) items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0)); - std::cout << fmt("%s%s: %s\n", indent, name, concatStringsSep(", ", items)); + logger->cout("%s%s: %s", indent, name, concatStringsSep(", ", items)); } } } @@ -106,7 +106,7 @@ using namespace nix; -struct CmdDiffClosures : SourceExprCommand +struct CmdDiffClosures : SourceExprCommand, MixOperateOnOptions { std::string _before, _after; diff -Nru nix-2.8.0/src/nix/doctor.cc nix-2.16.1+dfsg/src/nix/doctor.cc --- nix-2.8.0/src/nix/doctor.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/doctor.cc 2023-06-06 10:55:23.000000000 +0000 @@ -18,7 +18,7 @@ if (proto) { auto major = GET_PROTOCOL_MAJOR(proto) >> 8; auto minor = GET_PROTOCOL_MINOR(proto); - return (format("%1%.%2%") % major % minor).str(); + return fmt("%1%.%2%", major, minor); } return "unknown"; } @@ -33,12 +33,24 @@ return false; } +void checkInfo(const std::string & msg) { + notice(ANSI_BLUE "[INFO] " ANSI_NORMAL + msg); +} + } struct CmdDoctor : StoreCommand { bool success = true; + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + std::string description() override { return "check your system for potential problems and print a PASS or FAIL for each check"; @@ -55,6 +67,7 @@ success &= checkProfileRoots(store); } success &= checkStoreProtocol(store->getProtocol()); + checkTrustedUser(store); if (!success) throw Exit(2); @@ -130,6 +143,14 @@ return checkPass("Client protocol matches store protocol."); } + + void checkTrustedUser(ref store) + { + std::string_view trusted = store->isTrustedClient() + ? "trusted" + : "not trusted"; + checkInfo(fmt("You are %s by store uri: %s", trusted, store->getUri())); + } }; static auto rCmdDoctor = registerCommand("doctor"); diff -Nru nix-2.8.0/src/nix/edit.cc nix-2.16.1+dfsg/src/nix/edit.cc --- nix-2.8.0/src/nix/edit.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/edit.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,14 +1,15 @@ -#include "command.hh" +#include "command-installable-value.hh" #include "shared.hh" #include "eval.hh" #include "attr-path.hh" #include "progress-bar.hh" +#include "editor-for.hh" #include using namespace nix; -struct CmdEdit : InstallableCommand +struct CmdEdit : InstallableValueCommand { std::string description() override { @@ -24,23 +25,23 @@ Category category() override { return catSecondary; } - void run(ref store) override + void run(ref store, ref installable) override { auto state = getEvalState(); - auto [v, pos] = installable->toValue(*state); + const auto [file, line] = [&] { + auto [v, pos] = installable->toValue(*state); - try { - pos = findPackageFilename(*state, *v, installable->what()); - } catch (NoPositionInfo &) { - } - - if (pos == noPos) - throw Error("cannot find position information for '%s", installable->what()); + try { + return findPackageFilename(*state, *v, installable->what()); + } catch (NoPositionInfo &) { + throw Error("cannot find position information for '%s", installable->what()); + } + }(); stopProgressBar(); - auto args = editorFor(pos); + auto args = editorFor(file, line); restoreProcessContext(); diff -Nru nix-2.8.0/src/nix/eval.cc nix-2.16.1+dfsg/src/nix/eval.cc --- nix-2.8.0/src/nix/eval.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/eval.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,22 +1,23 @@ -#include "command.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" -#include "json.hh" #include "value-to-json.hh" #include "progress-bar.hh" +#include + using namespace nix; -struct CmdEval : MixJSON, InstallableCommand +struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption { bool raw = false; std::optional apply; std::optional writeTo; - CmdEval() : InstallableCommand(true /* supportReadOnlyMode */) + CmdEval() : InstallableValueCommand() { addFlag({ .longName = "raw", @@ -53,7 +54,7 @@ Category category() override { return catSecondary; } - void run(ref store) override + void run(ref store, ref installable) override { if (raw && json) throw UsageError("--raw and --json are mutually exclusive"); @@ -61,11 +62,11 @@ auto state = getEvalState(); auto [v, pos] = installable->toValue(*state); - PathSet context; + NixStringContext context; if (apply) { auto vApply = state->allocValue(); - state->eval(state->parseExprFromString(*apply, absPath(".")), *vApply); + state->eval(state->parseExprFromString(*apply, state->rootPath(CanonPath::fromCwd())), *vApply); auto vRes = state->allocValue(); state->callFunction(*vApply, *v, *vRes, noPos); v = vRes; @@ -77,9 +78,9 @@ if (pathExists(*writeTo)) throw Error("path '%s' already exists", *writeTo); - std::function recurse; + std::function recurse; - recurse = [&](Value & v, const Pos & pos, const Path & path) + recurse = [&](Value & v, const PosIdx pos, const Path & path) { state->forceValue(v, pos); if (v.type() == nString) @@ -88,18 +89,22 @@ else if (v.type() == nAttrs) { if (mkdir(path.c_str(), 0777) == -1) throw SysError("creating directory '%s'", path); - for (auto & attr : *v.attrs) + for (auto & attr : *v.attrs) { + std::string_view name = state->symbols[attr.name]; try { - if (attr.name == "." || attr.name == "..") - throw Error("invalid file name '%s'", attr.name); - recurse(*attr.value, *attr.pos, path + "/" + std::string(attr.name)); + if (name == "." || name == "..") + throw Error("invalid file name '%s'", name); + recurse(*attr.value, attr.pos, concatStrings(path, "/", name)); } catch (Error & e) { - e.addTrace(*attr.pos, hintfmt("while evaluating the attribute '%s'", attr.name)); + e.addTrace( + state->positions[attr.pos], + hintfmt("while evaluating the attribute '%s'", name)); throw; } + } } else - throw TypeError("value at '%s' is not a string or an attribute set", pos); + throw TypeError("value at '%s' is not a string or an attribute set", state->positions[pos]); }; recurse(*v, pos, *writeTo); @@ -107,17 +112,16 @@ else if (raw) { stopProgressBar(); - std::cout << *state->coerceToString(noPos, *v, context); + writeFull(STDOUT_FILENO, *state->coerceToString(noPos, *v, context, "while generating the eval command output")); } else if (json) { - JSONPlaceholder jsonOut(std::cout); - printValueAsJSON(*state, true, *v, pos, jsonOut, context); + logger->cout("%s", printValueAsJSON(*state, true, *v, pos, context, false)); } else { state->forceValueDeep(*v); - logger->cout("%s", *v); + logger->cout("%s", printValue(*state, *v)); } } }; diff -Nru nix-2.8.0/src/nix/eval.md nix-2.16.1+dfsg/src/nix/eval.md --- nix-2.8.0/src/nix/eval.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/eval.md 2023-06-06 10:55:23.000000000 +0000 @@ -18,7 +18,7 @@ * Evaluate a Nix expression from a file: ```console - # nix eval -f ./my-nixpkgs hello.name + # nix eval --file ./my-nixpkgs hello.name ``` * Get the current version of the `nixpkgs` flake: @@ -50,7 +50,7 @@ # Description -This command evaluates the Nix expression *installable* and prints the +This command evaluates the given Nix expression and prints the result on standard output. # Output format diff -Nru nix-2.8.0/src/nix/flake.cc nix-2.16.1+dfsg/src/nix/flake.cc --- nix-2.8.0/src/nix/flake.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/flake.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "eval.hh" @@ -7,11 +8,10 @@ #include "get-drvs.hh" #include "store-api.hh" #include "derivations.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "attr-path.hh" #include "fetchers.hh" #include "registry.hh" -#include "json.hh" #include "eval-cache.hh" #include "markdown.hh" @@ -21,6 +21,7 @@ using namespace nix; using namespace nix::flake; +using json = nlohmann::json; class FlakeCommand : virtual Args, public MixFlakeOptions { @@ -50,9 +51,9 @@ return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); } - std::optional getFlakeRefForCompletion() override + std::vector getFlakesForCompletion() override { - return getFlakeRef(); + return {flakeUrl}; } }; @@ -123,15 +124,15 @@ }; static void enumerateOutputs(EvalState & state, Value & vFlake, - std::function callback) + std::function callback) { auto pos = vFlake.determinePos(noPos); - state.forceAttrs(vFlake, pos); + state.forceAttrs(vFlake, pos, "while evaluating a flake to get its outputs"); auto aOutputs = vFlake.attrs->get(state.symbols.create("outputs")); assert(aOutputs); - state.forceAttrs(*aOutputs->value, pos); + state.forceAttrs(*aOutputs->value, pos, "while evaluating the outputs of a flake"); auto sHydraJobs = state.symbols.create("hydraJobs"); @@ -139,11 +140,11 @@ else. This way we can disable IFD for hydraJobs and then enable it for other outputs. */ if (auto attr = aOutputs->value->attrs->get(sHydraJobs)) - callback(attr->name, *attr->value, *attr->pos); + callback(state.symbols[attr->name], *attr->value, attr->pos); for (auto & attr : *aOutputs->value->attrs) { if (attr.name != sHydraJobs) - callback(attr.name, *attr.value, *attr.pos); + callback(state.symbols[attr.name], *attr.value, attr.pos); } } @@ -212,9 +213,10 @@ ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", std::put_time(std::localtime(&*lastModified), "%F %T")); - logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); + if (!lockedFlake.lockFile.root->inputs.empty()) + logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); - std::unordered_set> visited; + std::set> visited; std::function recurse; @@ -226,7 +228,7 @@ if (auto lockedNode = std::get_if<0>(&input.second)) { logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", prefix + (last ? treeLast : treeConn), input.first, - *lockedNode ? (*lockedNode)->lockedRef : flake.lockedRef); + (*lockedNode)->lockedRef); bool firstVisit = visited.insert(*lockedNode).second; @@ -254,17 +256,10 @@ } }; -static bool argHasName(std::string_view arg, std::string_view expected) -{ - return - arg == expected - || arg == "_" - || (hasPrefix(arg, "_") && arg.substr(1) == expected); -} - struct CmdFlakeCheck : FlakeCommand { bool build = true; + bool checkAllSystems = false; CmdFlakeCheck() { @@ -273,6 +268,11 @@ .description = "Do not build checks.", .handler = {&build, false} }); + addFlag({ + .longName = "all-systems", + .description = "Check the outputs for all systems.", + .handler = {&checkAllSystems, true} + }); } std::string description() override @@ -298,6 +298,7 @@ lockFlags.applyNixConfig = true; auto flake = lockFlake(); + auto localSystem = std::string(settings.thisSystem.get()); bool hasErrors = false; auto reportError = [&](const Error & e) { @@ -313,15 +314,38 @@ } }; + std::set omittedSystems; + // FIXME: rewrite to use EvalCache. - auto checkSystemName = [&](const std::string & system, const Pos & pos) { + auto resolve = [&] (PosIdx p) { + return state->positions[p]; + }; + + auto argHasName = [&] (Symbol arg, std::string_view expected) { + std::string_view name = state->symbols[arg]; + return + name == expected + || name == "_" + || (hasPrefix(name, "_") && name.substr(1) == expected); + }; + + auto checkSystemName = [&](const std::string & system, const PosIdx pos) { // FIXME: what's the format of "system"? if (system.find('-') == std::string::npos) - reportError(Error("'%s' is not a valid system type, at %s", system, pos)); + reportError(Error("'%s' is not a valid system type, at %s", system, resolve(pos))); }; - auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) -> std::optional { + auto checkSystemType = [&](const std::string & system, const PosIdx pos) { + if (!checkAllSystems && system != localSystem) { + omittedSystems.insert(system); + return false; + } else { + return true; + } + }; + + auto checkDerivation = [&](const std::string & attrPath, Value & v, const PosIdx pos) -> std::optional { try { auto drvInfo = getDerivation(*state, v, false); if (!drvInfo) @@ -329,7 +353,7 @@ // FIXME: check meta attributes return drvInfo->queryDrvPath(); } catch (Error & e) { - e.addTrace(pos, hintfmt("while checking the derivation '%s'", attrPath)); + e.addTrace(resolve(pos), hintfmt("while checking the derivation '%s'", attrPath)); reportError(e); } return std::nullopt; @@ -337,23 +361,23 @@ std::vector drvPaths; - auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { + auto checkApp = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { #if 0 // FIXME auto app = App(*state, v); for (auto & i : app.context) { - auto [drvPathS, outputName] = decodeContext(i); + auto [drvPathS, outputName] = NixStringContextElem::parse(i); store->parseStorePath(drvPathS); } #endif } catch (Error & e) { - e.addTrace(pos, hintfmt("while checking the app definition '%s'", attrPath)); + e.addTrace(resolve(pos), hintfmt("while checking the app definition '%s'", attrPath)); reportError(e); } }; - auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { + auto checkOverlay = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { state->forceValue(v, pos); if (!v.isLambda() @@ -368,118 +392,103 @@ // FIXME: if we have a 'nixpkgs' input, use it to // evaluate the overlay. } catch (Error & e) { - e.addTrace(pos, hintfmt("while checking the overlay '%s'", attrPath)); + e.addTrace(resolve(pos), hintfmt("while checking the overlay '%s'", attrPath)); reportError(e); } }; - auto checkModule = [&](const std::string & attrPath, Value & v, const Pos & pos) { + auto checkModule = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { state->forceValue(v, pos); - if (v.isLambda()) { - if (!v.lambda.fun->hasFormals() || !v.lambda.fun->formals->ellipsis) - throw Error("module must match an open attribute set ('{ config, ... }')"); - } else if (v.type() == nAttrs) { - for (auto & attr : *v.attrs) - try { - state->forceValue(*attr.value, *attr.pos); - } catch (Error & e) { - e.addTrace(*attr.pos, hintfmt("while evaluating the option '%s'", attr.name)); - throw; - } - } else - throw Error("module must be a function or an attribute set"); - // FIXME: if we have a 'nixpkgs' input, use it to - // check the module. } catch (Error & e) { - e.addTrace(pos, hintfmt("while checking the NixOS module '%s'", attrPath)); + e.addTrace(resolve(pos), hintfmt("while checking the NixOS module '%s'", attrPath)); reportError(e); } }; - std::function checkHydraJobs; + std::function checkHydraJobs; - checkHydraJobs = [&](const std::string & attrPath, Value & v, const Pos & pos) { + checkHydraJobs = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { - state->forceAttrs(v, pos); + state->forceAttrs(v, pos, ""); if (state->isDerivation(v)) throw Error("jobset should not be a derivation at top-level"); for (auto & attr : *v.attrs) { - state->forceAttrs(*attr.value, *attr.pos); - auto attrPath2 = attrPath + "." + (std::string) attr.name; + state->forceAttrs(*attr.value, attr.pos, ""); + auto attrPath2 = concatStrings(attrPath, ".", state->symbols[attr.name]); if (state->isDerivation(*attr.value)) { Activity act(*logger, lvlChatty, actUnknown, fmt("checking Hydra job '%s'", attrPath2)); - checkDerivation(attrPath2, *attr.value, *attr.pos); + checkDerivation(attrPath2, *attr.value, attr.pos); } else - checkHydraJobs(attrPath2, *attr.value, *attr.pos); + checkHydraJobs(attrPath2, *attr.value, attr.pos); } } catch (Error & e) { - e.addTrace(pos, hintfmt("while checking the Hydra jobset '%s'", attrPath)); + e.addTrace(resolve(pos), hintfmt("while checking the Hydra jobset '%s'", attrPath)); reportError(e); } }; - auto checkNixOSConfiguration = [&](const std::string & attrPath, Value & v, const Pos & pos) { + auto checkNixOSConfiguration = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlChatty, actUnknown, fmt("checking NixOS configuration '%s'", attrPath)); Bindings & bindings(*state->allocBindings(0)); auto vToplevel = findAlongAttrPath(*state, "config.system.build.toplevel", bindings, v).first; - state->forceAttrs(*vToplevel, pos); + state->forceValue(*vToplevel, pos); if (!state->isDerivation(*vToplevel)) throw Error("attribute 'config.system.build.toplevel' is not a derivation"); } catch (Error & e) { - e.addTrace(pos, hintfmt("while checking the NixOS configuration '%s'", attrPath)); + e.addTrace(resolve(pos), hintfmt("while checking the NixOS configuration '%s'", attrPath)); reportError(e); } }; - auto checkTemplate = [&](const std::string & attrPath, Value & v, const Pos & pos) { + auto checkTemplate = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlChatty, actUnknown, fmt("checking template '%s'", attrPath)); - state->forceAttrs(v, pos); + state->forceAttrs(v, pos, ""); if (auto attr = v.attrs->get(state->symbols.create("path"))) { if (attr->name == state->symbols.create("path")) { - PathSet context; - auto path = state->coerceToPath(*attr->pos, *attr->value, context); - if (!store->isInStore(path)) - throw Error("template '%s' has a bad 'path' attribute"); + NixStringContext context; + auto path = state->coerceToPath(attr->pos, *attr->value, context, ""); + if (!path.pathExists()) + throw Error("template '%s' refers to a non-existent path '%s'", attrPath, path); // TODO: recursively check the flake in 'path'. } } else throw Error("template '%s' lacks attribute 'path'", attrPath); if (auto attr = v.attrs->get(state->symbols.create("description"))) - state->forceStringNoCtx(*attr->value, *attr->pos); + state->forceStringNoCtx(*attr->value, attr->pos, ""); else throw Error("template '%s' lacks attribute 'description'", attrPath); for (auto & attr : *v.attrs) { - std::string name(attr.name); + std::string_view name(state->symbols[attr.name]); if (name != "path" && name != "description" && name != "welcomeText") throw Error("template '%s' has unsupported attribute '%s'", attrPath, name); } } catch (Error & e) { - e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); + e.addTrace(resolve(pos), hintfmt("while checking the template '%s'", attrPath)); reportError(e); } }; - auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) { + auto checkBundler = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { state->forceValue(v, pos); if (!v.isLambda()) throw Error("bundler must be a function"); // TODO: check types of inputs/outputs? } catch (Error & e) { - e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); + e.addTrace(resolve(pos), hintfmt("while checking the template '%s'", attrPath)); reportError(e); } }; @@ -492,7 +501,7 @@ enumerateOutputs(*state, *vFlake, - [&](const std::string & name, Value & vOutput, const Pos & pos) { + [&](const std::string & name, Value & vOutput, const PosIdx pos) { Activity act(*logger, lvlChatty, actUnknown, fmt("checking flake output '%s'", name)); @@ -503,7 +512,7 @@ std::string_view replacement = name == "defaultPackage" ? "packages..default" : - name == "defaultApps" ? "apps..default" : + name == "defaultApp" ? "apps..default" : name == "defaultTemplate" ? "templates.default" : name == "defaultBundler" ? "bundlers..default" : name == "overlay" ? "overlays.default" : @@ -514,78 +523,101 @@ warn("flake output attribute '%s' is deprecated; use '%s' instead", name, replacement); if (name == "checks") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { - checkSystemName(attr.name, *attr.pos); - state->forceAttrs(*attr.value, *attr.pos); - for (auto & attr2 : *attr.value->attrs) { - auto drvPath = checkDerivation( - fmt("%s.%s.%s", name, attr.name, attr2.name), - *attr2.value, *attr2.pos); - if (drvPath && (std::string) attr.name == settings.thisSystem.get()) - drvPaths.push_back(DerivedPath::Built{*drvPath}); + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs) { + auto drvPath = checkDerivation( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + if (drvPath && attr_name == settings.thisSystem.get()) { + drvPaths.push_back(DerivedPath::Built { + .drvPath = *drvPath, + .outputs = OutputsSpec::All { }, + }); + } + } } } } else if (name == "formatter") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { - checkSystemName(attr.name, *attr.pos); - checkApp( - fmt("%s.%s", name, attr.name), - *attr.value, *attr.pos); + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkApp( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } else if (name == "packages" || name == "devShells") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { - checkSystemName(attr.name, *attr.pos); - state->forceAttrs(*attr.value, *attr.pos); - for (auto & attr2 : *attr.value->attrs) - checkDerivation( - fmt("%s.%s.%s", name, attr.name, attr2.name), - *attr2.value, *attr2.pos); + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs) + checkDerivation( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + }; } } else if (name == "apps") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { - checkSystemName(attr.name, *attr.pos); - state->forceAttrs(*attr.value, *attr.pos); - for (auto & attr2 : *attr.value->attrs) - checkApp( - fmt("%s.%s.%s", name, attr.name, attr2.name), - *attr2.value, *attr2.pos); + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs) + checkApp( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + }; } } else if (name == "defaultPackage" || name == "devShell") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { - checkSystemName(attr.name, *attr.pos); - checkDerivation( - fmt("%s.%s", name, attr.name), - *attr.value, *attr.pos); + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkDerivation( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } else if (name == "defaultApp") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { - checkSystemName(attr.name, *attr.pos); - checkApp( - fmt("%s.%s", name, attr.name), - *attr.value, *attr.pos); + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos) ) { + checkApp( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } else if (name == "legacyPackages") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { - checkSystemName(attr.name, *attr.pos); + checkSystemName(state->symbols[attr.name], attr.pos); + checkSystemType(state->symbols[attr.name], attr.pos); // FIXME: do getDerivations? } } @@ -594,27 +626,27 @@ checkOverlay(name, vOutput, pos); else if (name == "overlays") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) - checkOverlay(fmt("%s.%s", name, attr.name), - *attr.value, *attr.pos); + checkOverlay(fmt("%s.%s", name, state->symbols[attr.name]), + *attr.value, attr.pos); } else if (name == "nixosModule") checkModule(name, vOutput, pos); else if (name == "nixosModules") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) - checkModule(fmt("%s.%s", name, attr.name), - *attr.value, *attr.pos); + checkModule(fmt("%s.%s", name, state->symbols[attr.name]), + *attr.value, attr.pos); } else if (name == "nixosConfigurations") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) - checkNixOSConfiguration(fmt("%s.%s", name, attr.name), - *attr.value, *attr.pos); + checkNixOSConfiguration(fmt("%s.%s", name, state->symbols[attr.name]), + *attr.value, attr.pos); } else if (name == "hydraJobs") @@ -624,40 +656,59 @@ checkTemplate(name, vOutput, pos); else if (name == "templates") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) - checkTemplate(fmt("%s.%s", name, attr.name), - *attr.value, *attr.pos); + checkTemplate(fmt("%s.%s", name, state->symbols[attr.name]), + *attr.value, attr.pos); } else if (name == "defaultBundler") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { - checkSystemName(attr.name, *attr.pos); - checkBundler( - fmt("%s.%s", name, attr.name), - *attr.value, *attr.pos); + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkBundler( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } else if (name == "bundlers") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { - checkSystemName(attr.name, *attr.pos); - state->forceAttrs(*attr.value, *attr.pos); - for (auto & attr2 : *attr.value->attrs) { - checkBundler( - fmt("%s.%s.%s", name, attr.name, attr2.name), - *attr2.value, *attr2.pos); - } + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs) { + checkBundler( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + } + }; } } + else if ( + name == "lib" + || name == "darwinConfigurations" + || name == "darwinModules" + || name == "flakeModule" + || name == "flakeModules" + || name == "herculesCI" + || name == "homeConfigurations" + || name == "nixopsConfigurations" + ) + // Known but unchecked community attribute + ; + else warn("unknown flake output '%s'", name); } catch (Error & e) { - e.addTrace(pos, hintfmt("while checking flake output '%s'", name)); + e.addTrace(resolve(pos), hintfmt("while checking flake output '%s'", name)); reportError(e); } }); @@ -669,7 +720,15 @@ } if (hasErrors) throw Error("some errors were encountered during the evaluation"); - } + + if (!omittedSystems.empty()) { + warn( + "The check omitted these incompatible systems: %s\n" + "Use '--all-systems' to check all.", + concatStringsSep(", ", omittedSystems) + ); + }; + }; }; static Strings defaultTemplateAttrPathsPrefixes{"templates."}; @@ -710,7 +769,7 @@ auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); auto installable = InstallableFlake(nullptr, - evalState, std::move(templateFlakeRef), templateName, + evalState, std::move(templateFlakeRef), templateName, DefaultOutputs(), defaultTemplateAttrPaths, defaultTemplateAttrPathsPrefixes, lockFlags); @@ -726,7 +785,8 @@ "If you've set '%s' to a string, try using a path instead.", templateDir, templateDirAttr->getAttrPathStr()); - std::vector files; + std::vector changedFiles; + std::vector conflictedFiles; std::function copyDir; copyDir = [&](const Path & from, const Path & to) @@ -743,31 +803,41 @@ auto contents = readFile(from2); if (pathExists(to2)) { auto contents2 = readFile(to2); - if (contents != contents2) - throw Error("refusing to overwrite existing file '%s'", to2); + if (contents != contents2) { + printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2, from2); + conflictedFiles.push_back(to2); + } else { + notice("skipping identical file: %s", from2); + } + continue; } else writeFile(to2, contents); } else if (S_ISLNK(st.st_mode)) { auto target = readLink(from2); if (pathExists(to2)) { - if (readLink(to2) != target) - throw Error("refusing to overwrite existing symlink '%s'", to2); + if (readLink(to2) != target) { + printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2, from2); + conflictedFiles.push_back(to2); + } else { + notice("skipping identical file: %s", from2); + } + continue; } else createSymlink(target, to2); } else throw Error("file '%s' has unsupported type", from2); - files.push_back(to2); + changedFiles.push_back(to2); notice("wrote: %s", to2); } }; copyDir(templateDir, flakeDir); - if (pathExists(flakeDir + "/.git")) { + if (!changedFiles.empty() && pathExists(flakeDir + "/.git")) { Strings args = { "-C", flakeDir, "add", "--intent-to-add", "--force", "--" }; - for (auto & s : files) args.push_back(s); + for (auto & s : changedFiles) args.push_back(s); runProgram("git", true, args); } auto welcomeText = cursor->maybeGetAttr("welcomeText"); @@ -775,6 +845,9 @@ notice("\n"); notice(renderMarkdownToTerminal(welcomeText->getString())); } + + if (!conflictedFiles.empty()) + throw Error("Encountered %d conflicts - see above", conflictedFiles.size()); } }; @@ -888,35 +961,44 @@ { auto flake = lockFlake(); - auto jsonRoot = json ? std::optional(std::cout) : std::nullopt; - StorePathSet sources; sources.insert(flake.flake.sourceInfo->storePath); - if (jsonRoot) - jsonRoot->attr("path", store->printStorePath(flake.flake.sourceInfo->storePath)); // FIXME: use graph output, handle cycles. - std::function & jsonObj)> traverse; - traverse = [&](const Node & node, std::optional & jsonObj) + std::function traverse; + traverse = [&](const Node & node) { - auto jsonObj2 = jsonObj ? jsonObj->object("inputs") : std::optional(); + nlohmann::json jsonObj2 = json ? json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { - auto jsonObj3 = jsonObj2 ? jsonObj2->object(inputName) : std::optional(); auto storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) : (*inputNode)->lockedRef.input.fetch(store).first.storePath; - if (jsonObj3) - jsonObj3->attr("path", store->printStorePath(storePath)); - sources.insert(std::move(storePath)); - traverse(**inputNode, jsonObj3); + if (json) { + auto& jsonObj3 = jsonObj2[inputName]; + jsonObj3["path"] = store->printStorePath(storePath); + sources.insert(std::move(storePath)); + jsonObj3["inputs"] = traverse(**inputNode); + } else { + sources.insert(std::move(storePath)); + traverse(**inputNode); + } } } + return jsonObj2; }; - traverse(*flake.lockFile.root, jsonRoot); + if (json) { + nlohmann::json jsonRoot = { + {"path", store->printStorePath(flake.flake.sourceInfo->storePath)}, + {"inputs", traverse(*flake.lockFile.root)}, + }; + logger->cout("%s", jsonRoot); + } else { + traverse(*flake.lockFile.root); + } if (!dryRun && !dstUri.empty()) { ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); @@ -928,6 +1010,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON { bool showLegacy = false; + bool showAllSystems = false; CmdFlakeShow() { @@ -936,6 +1019,11 @@ .description = "Show the contents of the `legacyPackages` output.", .handler = {&showLegacy, true} }); + addFlag({ + .longName = "all-systems", + .description = "Show the contents of outputs for all systems.", + .handler = {&showAllSystems, true} + }); } std::string description() override @@ -956,6 +1044,69 @@ auto state = getEvalState(); auto flake = std::make_shared(lockFlake()); + auto localSystem = std::string(settings.thisSystem.get()); + + std::function &attrPath, + const Symbol &attr)> hasContent; + + // For frameworks it's important that structures are as lazy as possible + // to prevent infinite recursions, performance issues and errors that + // aren't related to the thing to evaluate. As a consequence, they have + // to emit more attributes than strictly (sic) necessary. + // However, these attributes with empty values are not useful to the user + // so we omit them. + hasContent = [&]( + eval_cache::AttrCursor & visitor, + const std::vector &attrPath, + const Symbol &attr) -> bool + { + auto attrPath2(attrPath); + attrPath2.push_back(attr); + auto attrPathS = state->symbols.resolve(attrPath2); + const auto & attrName = state->symbols[attr]; + + auto visitor2 = visitor.getAttr(attrName); + + try { + if ((attrPathS[0] == "apps" + || attrPathS[0] == "checks" + || attrPathS[0] == "devShells" + || attrPathS[0] == "legacyPackages" + || attrPathS[0] == "packages") + && (attrPathS.size() == 1 || attrPathS.size() == 2)) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } + } + return false; + } + + if ((attrPathS.size() == 1) + && (attrPathS[0] == "formatter" + || attrPathS[0] == "nixosConfigurations" + || attrPathS[0] == "nixosModules" + || attrPathS[0] == "overlays" + )) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } + } + return false; + } + + // If we don't recognize it, it's probably content + return true; + } catch (EvalError & e) { + // Some attrs may contain errors, eg. legacyPackages of + // nixpkgs. We still want to recurse into it, instead of + // skipping it at all. + return true; + } + }; std::functionsymbols.resolve(attrPath); + Activity act(*logger, lvlInfo, actUnknown, - fmt("evaluating '%s'", concatStringsSep(".", attrPath))); + fmt("evaluating '%s'", concatStringsSep(".", attrPathS))); + try { auto recurse = [&]() { if (!json) logger->cout("%s", headerPrefix); - auto attrs = visitor.getAttrs(); + std::vector attrs; + for (const auto &attr : visitor.getAttrs()) { + if (hasContent(visitor, attrPath, attr)) + attrs.push_back(attr); + } + for (const auto & [i, attr] : enumerate(attrs)) { + const auto & attrName = state->symbols[attr]; bool last = i + 1 == attrs.size(); - auto visitor2 = visitor.getAttr(attr); + auto visitor2 = visitor.getAttr(attrName); auto attrPath2(attrPath); attrPath2.push_back(attr); auto j2 = visit(*visitor2, attrPath2, - fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, nextPrefix, last ? treeLast : treeConn, attr), + fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, nextPrefix, last ? treeLast : treeConn, attrName), nextPrefix + (last ? treeNull : treeLine)); - if (json) j.emplace(attr, std::move(j2)); + if (json) j.emplace(attrName, std::move(j2)); } }; @@ -997,8 +1157,8 @@ auto name = visitor.getAttr(state->sName)->getString(); if (json) { std::optional description; - if (auto aMeta = visitor.maybeGetAttr("meta")) { - if (auto aDescription = aMeta->maybeGetAttr("description")) + if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) { + if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) description = aDescription->getString(); } j.emplace("type", "derivation"); @@ -1008,10 +1168,10 @@ } else { logger->cout("%s: %s '%s'", headerPrefix, - attrPath.size() == 2 && attrPath[0] == "devShell" ? "development environment" : - attrPath.size() >= 2 && attrPath[0] == "devShells" ? "development environment" : - attrPath.size() == 3 && attrPath[0] == "checks" ? "derivation" : - attrPath.size() >= 1 && attrPath[0] == "hydraJobs" ? "derivation" : + attrPath.size() == 2 && attrPathS[0] == "devShell" ? "development environment" : + attrPath.size() >= 2 && attrPathS[0] == "devShells" ? "development environment" : + attrPath.size() == 3 && attrPathS[0] == "checks" ? "derivation" : + attrPath.size() >= 1 && attrPathS[0] == "hydraJobs" ? "derivation" : "package", name); } @@ -1019,48 +1179,66 @@ if (attrPath.size() == 0 || (attrPath.size() == 1 && ( - attrPath[0] == "defaultPackage" - || attrPath[0] == "devShell" - || attrPath[0] == "formatter" - || attrPath[0] == "nixosConfigurations" - || attrPath[0] == "nixosModules" - || attrPath[0] == "defaultApp" - || attrPath[0] == "templates" - || attrPath[0] == "overlays")) + attrPathS[0] == "defaultPackage" + || attrPathS[0] == "devShell" + || attrPathS[0] == "formatter" + || attrPathS[0] == "nixosConfigurations" + || attrPathS[0] == "nixosModules" + || attrPathS[0] == "defaultApp" + || attrPathS[0] == "templates" + || attrPathS[0] == "overlays")) || ((attrPath.size() == 1 || attrPath.size() == 2) - && (attrPath[0] == "checks" - || attrPath[0] == "packages" - || attrPath[0] == "devShells" - || attrPath[0] == "apps")) + && (attrPathS[0] == "checks" + || attrPathS[0] == "packages" + || attrPathS[0] == "devShells" + || attrPathS[0] == "apps")) ) { recurse(); } else if ( - (attrPath.size() == 2 && (attrPath[0] == "defaultPackage" || attrPath[0] == "devShell" || attrPath[0] == "formatter")) - || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages" || attrPath[0] == "devShells")) + (attrPath.size() == 2 && (attrPathS[0] == "defaultPackage" || attrPathS[0] == "devShell" || attrPathS[0] == "formatter")) + || (attrPath.size() == 3 && (attrPathS[0] == "checks" || attrPathS[0] == "packages" || attrPathS[0] == "devShells")) ) { - if (visitor.isDerivation()) - showDerivation(); - else - throw Error("expected a derivation"); + if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); + } + } else { + if (visitor.isDerivation()) + showDerivation(); + else + throw Error("expected a derivation"); + } } - else if (attrPath.size() > 0 && attrPath[0] == "hydraJobs") { + else if (attrPath.size() > 0 && attrPathS[0] == "hydraJobs") { if (visitor.isDerivation()) showDerivation(); else recurse(); } - else if (attrPath.size() > 0 && attrPath[0] == "legacyPackages") { + else if (attrPath.size() > 0 && attrPathS[0] == "legacyPackages") { if (attrPath.size() == 1) recurse(); - else if (!showLegacy) - logger->warn(fmt("%s: " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix)); - else { + else if (!showLegacy){ + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS))); + } + } else if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); + } + } else { if (visitor.isDerivation()) showDerivation(); else if (attrPath.size() <= 2) @@ -1070,8 +1248,8 @@ } else if ( - (attrPath.size() == 2 && attrPath[0] == "defaultApp") || - (attrPath.size() == 3 && attrPath[0] == "apps")) + (attrPath.size() == 2 && attrPathS[0] == "defaultApp") || + (attrPath.size() == 3 && attrPathS[0] == "apps")) { auto aType = visitor.maybeGetAttr("type"); if (!aType || aType->getString() != "app") @@ -1084,8 +1262,8 @@ } else if ( - (attrPath.size() == 1 && attrPath[0] == "defaultTemplate") || - (attrPath.size() == 2 && attrPath[0] == "templates")) + (attrPath.size() == 1 && attrPathS[0] == "defaultTemplate") || + (attrPath.size() == 2 && attrPathS[0] == "templates")) { auto description = visitor.getAttr("description")->getString(); if (json) { @@ -1098,11 +1276,11 @@ else { auto [type, description] = - (attrPath.size() == 1 && attrPath[0] == "overlay") - || (attrPath.size() == 2 && attrPath[0] == "overlays") ? std::make_pair("nixpkgs-overlay", "Nixpkgs overlay") : - attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? std::make_pair("nixos-configuration", "NixOS configuration") : - (attrPath.size() == 1 && attrPath[0] == "nixosModule") - || (attrPath.size() == 2 && attrPath[0] == "nixosModules") ? std::make_pair("nixos-module", "NixOS module") : + (attrPath.size() == 1 && attrPathS[0] == "overlay") + || (attrPath.size() == 2 && attrPathS[0] == "overlays") ? std::make_pair("nixpkgs-overlay", "Nixpkgs overlay") : + attrPath.size() == 2 && attrPathS[0] == "nixosConfigurations" ? std::make_pair("nixos-configuration", "NixOS configuration") : + (attrPath.size() == 1 && attrPathS[0] == "nixosModule") + || (attrPath.size() == 2 && attrPathS[0] == "nixosModules") ? std::make_pair("nixos-module", "NixOS module") : std::make_pair("unknown", "unknown"); if (json) { j.emplace("type", type); @@ -1111,7 +1289,7 @@ } } } catch (EvalError & e) { - if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages")) + if (!(attrPath.size() > 0 && attrPathS[0] == "legacyPackages")) throw; } @@ -1200,8 +1378,7 @@ { if (!command) throw UsageError("'nix flake' requires a sub-command."); - settings.requireExperimentalFeature(Xp::Flakes); - command->second->prepare(); + experimentalFeatureSettings.require(Xp::Flakes); command->second->run(); } }; diff -Nru nix-2.8.0/src/nix/flake-check.md nix-2.16.1+dfsg/src/nix/flake-check.md --- nix-2.8.0/src/nix/flake-check.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/flake-check.md 2023-06-06 10:55:23.000000000 +0000 @@ -68,6 +68,6 @@ In addition, the `hydraJobs` output is evaluated in the same way as Hydra's `hydra-eval-jobs` (i.e. as a arbitrarily deeply nested attribute set of derivations). Similarly, the -`legacyPackages`.*system* output is evaluated like `nix-env -qa`. +`legacyPackages`.*system* output is evaluated like `nix-env --query --available `. )"" diff -Nru nix-2.8.0/src/nix/flake.md nix-2.16.1+dfsg/src/nix/flake.md --- nix-2.8.0/src/nix/flake.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/flake.md 2023-06-06 10:55:23.000000000 +0000 @@ -18,51 +18,56 @@ Flake references (*flakerefs*) are a way to specify the location of a flake. These have two different forms: -* An attribute set representation, e.g. - ```nix - { - type = "github"; - owner = "NixOS"; - repo = "nixpkgs"; - } - ``` +## Attribute set representation - The only required attribute is `type`. The supported types are - listed below. +Example: -* A URL-like syntax, e.g. +```nix +{ + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; +} +``` - ``` - github:NixOS/nixpkgs - ``` +The only required attribute is `type`. The supported types are +listed below. - These are used on the command line as a more convenient alternative - to the attribute set representation. For instance, in the command +## URL-like syntax - ```console - # nix build github:NixOS/nixpkgs#hello - ``` +Example: - `github:NixOS/nixpkgs` is a flake reference (while `hello` is an - output attribute). They are also allowed in the `inputs` attribute - of a flake, e.g. +``` +github:NixOS/nixpkgs +``` - ```nix - inputs.nixpkgs.url = github:NixOS/nixpkgs; - ``` +These are used on the command line as a more convenient alternative +to the attribute set representation. For instance, in the command - is equivalent to +```console +# nix build github:NixOS/nixpkgs#hello +``` - ```nix - inputs.nixpkgs = { - type = "github"; - owner = "NixOS"; - repo = "nixpkgs"; - }; - ``` +`github:NixOS/nixpkgs` is a flake reference (while `hello` is an +output attribute). They are also allowed in the `inputs` attribute +of a flake, e.g. + +```nix +inputs.nixpkgs.url = "github:NixOS/nixpkgs"; +``` -## Examples +is equivalent to + +```nix +inputs.nixpkgs = { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; +}; +``` + +### Examples Here are some examples of flake references in their URL-like representation: @@ -153,7 +158,7 @@ git(+http|+https|+ssh|+git|+file|):(//)?(\?)? ``` - The `ref` attribute defaults to `master`. + The `ref` attribute defaults to resolving the `HEAD` reference. The `rev` attribute must denote a commit that exists in the branch or tag specified by the `ref` attribute, since Nix doesn't do a full @@ -161,6 +166,11 @@ doesn't allow fetching a `rev` without a known `ref`). The default is the commit currently pointed to by `ref`. + When `git+file` is used without specifying `ref` or `rev`, files are + fetched directly from the local `path` as long as they have been added + to the Git repository. If there are uncommitted changes, the reference + is treated as dirty and a warning is printed. + For example, the following are valid Git flake references: * `git+https://example.org/my/repo` @@ -176,9 +186,17 @@ * `tarball`: Tarballs. The location of the tarball is specified by the attribute `url`. - In URL form, the schema must be `http://`, `https://` or `file://` - URLs and the extension must be `.zip`, `.tar`, `.tgz`, `.tar.gz`, - `.tar.xz`, `.tar.bz2` or `.tar.zst`. + In URL form, the schema must be `tarball+http://`, `tarball+https://` or `tarball+file://`. + If the extension corresponds to a known archive format (`.zip`, `.tar`, + `.tgz`, `.tar.gz`, `.tar.xz`, `.tar.bz2` or `.tar.zst`), then the `tarball+` + can be dropped. + +* `file`: Plain files or directory tarballs, either over http(s) or from the local + disk. + + In URL form, the schema must be `file+http://`, `file+https://` or `file+file://`. + If the extension doesn’t correspond to a known archive format (as defined by the + `tarball` fetcher), then the `file+` prefix can be dropped. * `github`: A more efficient way to fetch repositories from GitHub. The following attributes are required: @@ -203,11 +221,46 @@ commit hash (`rev`). Note that unlike Git, GitHub allows fetching by commit hash without specifying a branch or tag. + You can also specify `host` as a parameter, to point to a custom GitHub + Enterprise server. + Some examples: * `github:edolstra/dwarffs` * `github:edolstra/dwarffs/unstable` * `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31` + * `github:internal/project?host=company-github.example.org` + +* `gitlab`: Similar to `github`, is a more efficient way to fetch + GitLab repositories. The following attributes are required: + + * `owner`: The owner of the repository. + + * `repo`: The name of the repository. + + Like `github`, these are downloaded as tarball archives. + + The URL syntax for `gitlab` flakes is: + + `gitlab:/(/)?(\?)?` + + `` works the same as `github`. Either a branch or tag name + (`ref`), or a commit hash (`rev`) can be specified. + + Since GitLab allows for self-hosting, you can specify `host` as + a parameter, to point to any instances other than `gitlab.com`. + + Some examples: + + * `gitlab:veloren/veloren` + * `gitlab:veloren/veloren/master` + * `gitlab:veloren/veloren/80a4d7f13492d916e47d6195be23acae8001985a` + * `gitlab:openldap/openldap?host=git.openldap.org` + + When accessing a project in a (nested) subgroup, make sure to URL-encode any + slashes, i.e. replace `/` with `%2F`: + + * `gitlab:veloren%2Fdev/rfcs` * `sourcehut`: Similar to `github`, is a more efficient way to fetch SourceHut repositories. The following attributes are required: @@ -257,14 +310,14 @@ # Flake format As an example, here is a simple `flake.nix` that depends on the -Nixpkgs flake and provides a single package (i.e. an installable -derivation): +Nixpkgs flake and provides a single package (i.e. an +[installable](./nix.md#installables) derivation): ```nix { description = "A flake for building Hello World"; - inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-20.03; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-20.03"; outputs = { self, nixpkgs }: { @@ -299,6 +352,8 @@ also contains some metadata about the inputs. These are: * `outPath`: The path in the Nix store of the flake's source tree. + This way, the attribute set can be passed to `import` as if it was a path, + as in the example above (`import nixpkgs`). * `rev`: The commit hash of the flake's repository, if applicable. @@ -326,9 +381,12 @@ * `nixConfig`: a set of `nix.conf` options to be set when evaluating any part of a flake. In the interests of security, only a small set of - whitelisted options (currently `bash-prompt`, `bash-prompt-suffix`, - and `flake-registry`) are allowed to be set without confirmation so long as - `accept-flake-config` is not set in the global configuration. + set of options is allowed to be set without confirmation so long as [`accept-flake-config`](@docroot@/command-ref/conf-file.md#conf-accept-flake-config) is not enabled in the global configuration: + - [`bash-prompt`](@docroot@/command-ref/conf-file.md#conf-bash-prompt) + - [`bash-prompt-prefix`](@docroot@/command-ref/conf-file.md#conf-bash-prompt-prefix) + - [`bash-prompt-suffix`](@docroot@/command-ref/conf-file.md#conf-bash-prompt-suffix) + - [`flake-registry`](@docroot@/command-ref/conf-file.md#conf-flake-registry) + - [`commit-lockfile-summary`](@docroot@/command-ref/conf-file.md#conf-commit-lockfile-summary) ## Flake inputs @@ -355,7 +413,7 @@ Alternatively, you can use the URL-like syntax: ```nix -inputs.import-cargo.url = github:edolstra/import-cargo; +inputs.import-cargo.url = "github:edolstra/import-cargo"; inputs.nixpkgs.url = "nixpkgs"; ``` diff -Nru nix-2.8.0/src/nix/flake-update.md nix-2.16.1+dfsg/src/nix/flake-update.md --- nix-2.8.0/src/nix/flake-update.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/flake-update.md 2023-06-06 10:55:23.000000000 +0000 @@ -6,7 +6,7 @@ lock file: ```console - # nix flake update + # nix flake update --commit-lock-file * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' … @@ -16,7 +16,7 @@ # Description This command recreates the lock file of a flake (`flake.lock`), thus -updating the lock for every mutable input (like `nixpkgs`) to its +updating the lock for every unlocked input (like `nixpkgs`) to its current version. This is equivalent to passing `--recreate-lock-file` to any command that operates on a flake. That is, diff -Nru nix-2.8.0/src/nix/fmt.cc nix-2.16.1+dfsg/src/nix/fmt.cc --- nix-2.8.0/src/nix/fmt.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/fmt.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-value.hh" #include "run.hh" using namespace nix; @@ -26,12 +27,14 @@ Strings getDefaultFlakeAttrPathPrefixes() override { return Strings{}; } - void run(ref store) { + void run(ref store) override + { auto evalState = getEvalState(); auto evalStore = getEvalStore(); - auto installable = parseInstallable(store, "."); - auto app = installable->toApp(*evalState).resolve(evalStore, store); + auto installable_ = parseInstallable(store, "."); + auto & installable = InstallableValue::require(*installable_); + auto app = installable.toApp(*evalState).resolve(evalStore, store); Strings programArgs{app.program}; diff -Nru nix-2.8.0/src/nix/get-env.sh nix-2.16.1+dfsg/src/nix/get-env.sh --- nix-2.8.0/src/nix/get-env.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/get-env.sh 2023-06-06 10:55:23.000000000 +0000 @@ -43,6 +43,7 @@ local __var_name="${BASH_REMATCH[2]}" if [[ $__var_name =~ ^BASH_ || \ + $__var_name =~ ^COMP_ || \ $__var_name = _ || \ $__var_name = DIRSTACK || \ $__var_name = EUID || \ @@ -54,7 +55,9 @@ $__var_name = PWD || \ $__var_name = RANDOM || \ $__var_name = SHLVL || \ - $__var_name = SECONDS \ + $__var_name = SECONDS || \ + $__var_name = EPOCHREALTIME || \ + $__var_name = EPOCHSECONDS \ ]]; then continue; fi if [[ -z $__first ]]; then printf ',\n'; else __first=; fi diff -Nru nix-2.8.0/src/nix/hash.cc nix-2.16.1+dfsg/src/nix/hash.cc --- nix-2.8.0/src/nix/hash.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/hash.cc 2023-06-06 10:55:23.000000000 +0000 @@ -151,7 +151,6 @@ { if (!command) throw UsageError("'nix hash' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; @@ -161,11 +160,11 @@ /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) { - HashType ht = htMD5; + std::optional ht; bool flat = false; - bool base32 = false; + Base base = Base16; bool truncate = false; - enum { opHash, opTo32, opTo16 } op = opHash; + enum { opHash, opTo } op = opHash; std::vector ss; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { @@ -174,14 +173,31 @@ else if (*arg == "--version") printVersion("nix-hash"); else if (*arg == "--flat") flat = true; - else if (*arg == "--base32") base32 = true; + else if (*arg == "--base16") base = Base16; + else if (*arg == "--base32") base = Base32; + else if (*arg == "--base64") base = Base64; + else if (*arg == "--sri") base = SRI; else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); ht = parseHashType(s); } - else if (*arg == "--to-base16") op = opTo16; - else if (*arg == "--to-base32") op = opTo32; + else if (*arg == "--to-base16") { + op = opTo; + base = Base16; + } + else if (*arg == "--to-base32") { + op = opTo; + base = Base32; + } + else if (*arg == "--to-base64") { + op = opTo; + base = Base64; + } + else if (*arg == "--to-sri") { + op = opTo; + base = SRI; + } else if (*arg != "" && arg->at(0) == '-') return false; else @@ -191,17 +207,18 @@ if (op == opHash) { CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); - cmd.ht = ht; - cmd.base = base32 ? Base32 : Base16; + if (!ht.has_value()) ht = htMD5; + cmd.ht = ht.value(); + cmd.base = base; cmd.truncate = truncate; cmd.paths = ss; cmd.run(); } else { - CmdToBase cmd(op == opTo32 ? Base32 : Base16); + CmdToBase cmd(base); cmd.args = ss; - cmd.ht = ht; + if (ht.has_value()) cmd.ht = ht; cmd.run(); } diff -Nru nix-2.8.0/src/nix/help-stores.md nix-2.16.1+dfsg/src/nix/help-stores.md --- nix-2.8.0/src/nix/help-stores.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/help-stores.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,46 @@ +R"( + +Nix supports different types of stores. These are described below. + +## Store URL format + +Stores are specified using a URL-like syntax. For example, the command + +```console +# nix path-info --store https://cache.nixos.org/ --json \ + /nix/store/a7gvj343m05j2s32xcnwr35v31ynlypr-coreutils-9.1 +``` + +fetches information about a store path in the HTTP binary cache +located at https://cache.nixos.org/, which is a type of store. + +Store URLs can specify **store settings** using URL query strings, +i.e. by appending `?name1=value1&name2=value2&...` to the URL. For +instance, + +``` +--store ssh://machine.example.org?ssh-key=/path/to/my/key +``` + +tells Nix to access the store on a remote machine via the SSH +protocol, using `/path/to/my/key` as the SSH private key. The +supported settings for each store type are documented below. + +The special store URL `auto` causes Nix to automatically select a +store as follows: + +* Use the [local store](#local-store) `/nix/store` if `/nix/var/nix` + is writable by the current user. + +* Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect + to the Nix daemon listening on that socket](#local-daemon-store). + +* Otherwise, on Linux only, use the [local chroot store](#local-store) + `~/.local/share/nix/root`, which will be created automatically if it + does not exist. + +* Otherwise, use the [local store](#local-store) `/nix/store`. + +@stores@ + +)" diff -Nru nix-2.8.0/src/nix/key-generate-secret.md nix-2.16.1+dfsg/src/nix/key-generate-secret.md --- nix-2.8.0/src/nix/key-generate-secret.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/key-generate-secret.md 2023-06-06 10:55:23.000000000 +0000 @@ -30,7 +30,7 @@ verifying signed store paths. The mandatory argument `--key-name` specifies a key name (such as -`cache.example.org-1). It is used to look up keys on the client when +`cache.example.org-1`). It is used to look up keys on the client when it verifies signatures. It can be anything, but it’s suggested to use the host name of your cache (e.g. `cache.example.org`) with a suffix denoting the number of the key (to be incremented every time you need diff -Nru nix-2.8.0/src/nix/local.mk nix-2.16.1+dfsg/src/nix/local.mk --- nix-2.8.0/src/nix/local.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -18,7 +18,7 @@ nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ @@ -32,3 +32,9 @@ src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh + +src/nix/doc/files/%.md: doc/manual/src/command-ref/files/%.md + @mkdir -p $$(dirname $@) + @cp $< $@ + +src/nix/profile.cc: src/nix/profile.md src/nix/doc/files/profiles.md.gen.hh diff -Nru nix-2.8.0/src/nix/log.cc nix-2.16.1+dfsg/src/nix/log.cc --- nix-2.8.0/src/nix/log.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/log.cc 2023-06-06 10:55:23.000000000 +0000 @@ -23,7 +23,7 @@ Category category() override { return catSecondary; } - void run(ref store) override + void run(ref store, ref installable) override { settings.readOnlyMode = true; @@ -49,11 +49,11 @@ [&](const DerivedPath::Built & bfd) { return logSub.getBuildLog(bfd.drvPath); }, - }, b.raw()); + }, b.path.raw()); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); - std::cout << *log; + writeFull(STDOUT_FILENO, *log); return; } diff -Nru nix-2.8.0/src/nix/log.md nix-2.16.1+dfsg/src/nix/log.md --- nix-2.8.0/src/nix/log.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/log.md 2023-06-06 10:55:23.000000000 +0000 @@ -22,8 +22,7 @@ # Description -This command prints the log of a previous build of the derivation -*installable* on standard output. +This command prints the log of a previous build of the [*installable*](./nix.md#installables) on standard output. Nix looks for build logs in two places: diff -Nru nix-2.8.0/src/nix/ls.cc nix-2.16.1+dfsg/src/nix/ls.cc --- nix-2.8.0/src/nix/ls.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/ls.cc 2023-06-06 10:55:23.000000000 +0000 @@ -3,7 +3,7 @@ #include "fs-accessor.hh" #include "nar-accessor.hh" #include "common-args.hh" -#include "json.hh" +#include using namespace nix; @@ -91,10 +91,9 @@ if (path == "/") path = ""; if (json) { - JSONPlaceholder jsonRoot(std::cout); if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - listNar(jsonRoot, accessor, path, recursive); + logger->cout("%s", listNar(accessor, path, recursive)); } else listText(accessor); } diff -Nru nix-2.8.0/src/nix/main.cc nix-2.16.1+dfsg/src/nix/main.cc --- nix-2.8.0/src/nix/main.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/main.cc 2023-06-06 10:55:23.000000000 +0000 @@ -53,19 +53,18 @@ } std::string programPath; -char * * savedArgv; - -struct HelpRequested { }; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool useNet = true; bool refresh = false; + bool helpRequested = false; bool showVersion = false; NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") { categories.clear(); + categories[catHelp] = "Help commands"; categories[Command::catDefault] = "Main commands"; categories[catSecondary] = "Infrequently used commands"; categories[catUtility] = "Utility/scripting commands"; @@ -74,7 +73,8 @@ addFlag({ .longName = "help", .description = "Show usage information.", - .handler = {[&]() { throw HelpRequested(); }}, + .category = miscCategory, + .handler = {[this]() { this->helpRequested = true; }}, }); addFlag({ @@ -82,12 +82,14 @@ .shortName = 'L', .description = "Print full build logs on standard error.", .category = loggingCategory, - .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, + .handler = {[&]() { logger->setPrintBuildLogs(true); }}, + .experimentalFeature = Xp::NixCommand, }); addFlag({ .longName = "version", .description = "Show version information.", + .category = miscCategory, .handler = {[&]() { showVersion = true; }}, }); @@ -95,13 +97,17 @@ .longName = "offline", .aliases = {"no-net"}, // FIXME: remove .description = "Disable substituters and consider all previously downloaded files up-to-date.", + .category = miscCategory, .handler = {[&]() { useNet = false; }}, + .experimentalFeature = Xp::NixCommand, }); addFlag({ .longName = "refresh", .description = "Consider all previously downloaded files out-of-date.", + .category = miscCategory, .handler = {[&]() { refresh = true; }}, + .experimentalFeature = Xp::NixCommand, }); } @@ -121,6 +127,7 @@ {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, + {"show-derivation", {"derivation", "show"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, @@ -161,11 +168,29 @@ { commands = RegisterCommand::getCommandsFor({}); } + + std::string dumpCli() + { + auto res = nlohmann::json::object(); + + res["args"] = toJSON(); + + auto stores = nlohmann::json::object(); + for (auto & implem : *Implementations::registered) { + auto storeConfig = implem.getConfig(); + auto storeName = storeConfig->name(); + stores[storeName]["doc"] = storeConfig->doc(); + stores[storeName]["settings"] = storeConfig->toJSON(); + } + res["stores"] = std::move(stores); + + return res.dump(); + } }; /* Render the help for the specified subcommand to stdout using lowdown. */ -static void showHelp(std::vector subcommand, MultiCommand & toplevel) +static void showHelp(std::vector subcommand, NixArgs & toplevel) { auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand)); @@ -176,32 +201,40 @@ auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( #include "generate-manpage.nix.gen.hh" - , "/"), *vGenerateManpage); + , CanonPath::root), *vGenerateManpage); auto vUtils = state.allocValue(); state.cacheFile( - "/utils.nix", "/utils.nix", + CanonPath("/utils.nix"), CanonPath("/utils.nix"), state.parseExprFromString( #include "utils.nix.gen.hh" - , "/"), + , CanonPath::root), *vUtils); - auto attrs = state.buildBindings(16); - attrs.alloc("command").mkString(toplevel.toJSON().dump()); + auto vDump = state.allocValue(); + vDump->mkString(toplevel.dumpCli()); auto vRes = state.allocValue(); - state.callFunction(*vGenerateManpage, state.allocValue()->mkAttrs(attrs), *vRes, noPos); + state.callFunction(*vGenerateManpage, *vDump, *vRes, noPos); auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); if (!attr) throw UsageError("Nix has no subcommand '%s'", concatStringsSep("", subcommand)); - auto markdown = state.forceString(*attr->value); + auto markdown = state.forceString(*attr->value, noPos, "while evaluating the lowdown help text"); RunPager pager; std::cout << renderMarkdownToTerminal(markdown) << "\n"; } +static NixArgs & getNixArgs(Command & cmd) +{ + assert(cmd.parent); + MultiCommand * toplevel = cmd.parent; + while (toplevel->parent) toplevel = toplevel->parent; + return dynamic_cast(*toplevel); +} + struct CmdHelp : Command { std::vector subcommand; @@ -226,17 +259,43 @@ ; } + Category category() override { return catHelp; } + void run() override { assert(parent); MultiCommand * toplevel = parent; while (toplevel->parent) toplevel = toplevel->parent; - showHelp(subcommand, *toplevel); + showHelp(subcommand, getNixArgs(*this)); } }; static auto rCmdHelp = registerCommand("help"); +struct CmdHelpStores : Command +{ + std::string description() override + { + return "show help about store types and their settings"; + } + + std::string doc() override + { + return + #include "help-stores.md" + ; + } + + Category category() override { return catHelp; } + + void run() override + { + showHelp({"help-stores"}, getNixArgs(*this)); + } +}; + +static auto rCmdHelpStores = registerCommand("help-stores"); + void mainWrapped(int argc, char * * argv) { savedArgv = argv; @@ -261,9 +320,16 @@ } #endif + Finally f([] { logger->stop(); }); + programPath = argv[0]; auto programName = std::string(baseNameOf(programPath)); + if (argc > 1 && std::string_view(argv[1]) == "__build-remote") { + programName = "build-remote"; + argv++; argc--; + } + { auto legacy = (*RegisterLegacyCommand::commands)[programName]; if (legacy) return legacy(argc, argv); @@ -279,17 +345,18 @@ verbosity = lvlInfo; } - Finally f([] { logger->stop(); }); - NixArgs args; - if (argc == 2 && std::string(argv[1]) == "__dump-args") { - std::cout << args.toJSON().dump() << "\n"; + if (argc == 2 && std::string(argv[1]) == "__dump-cli") { + logger->cout(args.dumpCli()); return; } if (argc == 2 && std::string(argv[1]) == "__dump-builtins") { - settings.experimentalFeatures = {Xp::Flakes, Xp::FetchClosure}; + experimentalFeatureSettings.experimentalFeatures = { + Xp::Flakes, + Xp::FetchClosure, + }; evalSettings.pureEval = false; EvalState state({}, openStore("dummy://")); auto res = nlohmann::json::object(); @@ -302,9 +369,14 @@ b["arity"] = primOp->arity; b["args"] = primOp->args; b["doc"] = trim(stripIndentation(primOp->doc)); - res[(std::string) builtin.name] = std::move(b); + res[state.symbols[builtin.name]] = std::move(b); } - std::cout << res.dump() << "\n"; + logger->cout("%s", res); + return; + } + + if (argc == 2 && std::string(argv[1]) == "__dump-xp-features") { + logger->cout(documentExperimentalFeatures().dump()); return; } @@ -313,20 +385,24 @@ if (completions) { switch (completionType) { case ctNormal: - std::cout << "normal\n"; break; + logger->cout("normal"); break; case ctFilenames: - std::cout << "filenames\n"; break; + logger->cout("filenames"); break; case ctAttrs: - std::cout << "attrs\n"; break; + logger->cout("attrs"); break; } for (auto & s : *completions) - std::cout << s.completion << "\t" << s.description << "\n"; + logger->cout(s.completion + "\t" + trim(s.description)); } }); try { args.parseCmdline(argvToStrings(argc, argv)); - } catch (HelpRequested &) { + } catch (UsageError &) { + if (!args.helpRequested && !completions) throw; + } + + if (args.helpRequested) { std::vector subcommand; MultiCommand * command = &args; while (command) { @@ -338,11 +414,12 @@ } showHelp(subcommand, args); return; - } catch (UsageError &) { - if (!completions) throw; } - if (completions) return; + if (completions) { + args.completionHook(); + return; + } if (args.showVersion) { printVersion(programName); @@ -352,10 +429,8 @@ if (!args.command) throw UsageError("no subcommand specified"); - if (args.command->first != "repl" - && args.command->first != "doctor" - && args.command->first != "upgrade-nix") - settings.requireExperimentalFeature(Xp::NixCommand); + experimentalFeatureSettings.require( + args.command->second->experimentalFeature()); if (args.useNet && !haveInternet()) { warn("you don't have Internet access; disabling some network-dependent features"); @@ -380,7 +455,9 @@ settings.ttlPositiveNarInfoCache = 0; } - args.command->second->prepare(); + if (args.command->second->forceImpureByDefault() && !evalSettings.pureEval.overridden) { + evalSettings.pureEval = false; + } args.command->second->run(); } diff -Nru nix-2.8.0/src/nix/make-content-addressed.cc nix-2.16.1+dfsg/src/nix/make-content-addressed.cc --- nix-2.8.0/src/nix/make-content-addressed.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/make-content-addressed.cc 2023-06-06 10:55:23.000000000 +0000 @@ -2,10 +2,13 @@ #include "store-api.hh" #include "make-content-addressed.hh" #include "common-args.hh" -#include "json.hh" + +#include using namespace nix; +using nlohmann::json; + struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON { CmdMakeContentAddressed() @@ -33,13 +36,15 @@ StorePathSet(storePaths.begin(), storePaths.end())); if (json) { - JSONObject jsonRoot(std::cout); - JSONObject jsonRewrites(jsonRoot.object("rewrites")); + auto jsonRewrites = json::object(); for (auto & path : storePaths) { auto i = remappings.find(path); assert(i != remappings.end()); - jsonRewrites.attr(srcStore->printStorePath(path), srcStore->printStorePath(i->second)); + jsonRewrites[srcStore->printStorePath(path)] = srcStore->printStorePath(i->second); } + auto json = json::object(); + json["rewrites"] = jsonRewrites; + logger->cout("%s", json); } else { for (auto & path : storePaths) { auto i = remappings.find(path); diff -Nru nix-2.8.0/src/nix/make-content-addressed.md nix-2.16.1+dfsg/src/nix/make-content-addressed.md --- nix-2.8.0/src/nix/make-content-addressed.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/make-content-addressed.md 2023-06-06 10:55:23.000000000 +0000 @@ -22,7 +22,7 @@ ```console # nix copy --to /tmp/nix --trusted-public-keys '' nixpkgs#hello - cannot add path '/nix/store/zy9wbxwcygrwnh8n2w9qbbcr6zk87m26-libunistring-0.9.10' because it lacks a valid signature + cannot add path '/nix/store/zy9wbxwcygrwnh8n2w9qbbcr6zk87m26-libunistring-0.9.10' because it lacks a signature by a trusted key ``` * Create a content-addressed representation of the current NixOS @@ -35,7 +35,9 @@ # Description This command converts the closure of the store paths specified by -*installables* to content-addressed form. Nix store paths are usually +[*installables*](./nix.md#installables) to content-addressed form. + +Nix store paths are usually *input-addressed*, meaning that the hash part of the store path is computed from the contents of the derivation (i.e., the build-time dependency graph). Input-addressed paths need to be signed by a diff -Nru nix-2.8.0/src/nix/nar.cc nix-2.16.1+dfsg/src/nix/nar.cc --- nix-2.8.0/src/nix/nar.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/nar.cc 2023-06-06 10:55:23.000000000 +0000 @@ -25,7 +25,6 @@ { if (!command) throw UsageError("'nix nar' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; diff -Nru nix-2.8.0/src/nix/nar-ls.md nix-2.16.1+dfsg/src/nix/nar-ls.md --- nix-2.8.0/src/nix/nar-ls.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/nar-ls.md 2023-06-06 10:55:23.000000000 +0000 @@ -5,7 +5,7 @@ * To list a specific file in a NAR: ```console - # nix nar ls -l ./hello.nar /bin/hello + # nix nar ls --long ./hello.nar /bin/hello -r-xr-xr-x 38184 hello ``` @@ -13,7 +13,7 @@ format: ```console - # nix nar ls --json -R ./hello.nar /bin + # nix nar ls --json --recursive ./hello.nar /bin {"type":"directory","entries":{"hello":{"type":"regular","size":38184,"executable":true,"narOffset":400}}} ``` diff -Nru nix-2.8.0/src/nix/nix.md nix-2.16.1+dfsg/src/nix/nix.md --- nix-2.8.0/src/nix/nix.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/nix.md 2023-06-06 10:55:23.000000000 +0000 @@ -48,108 +48,191 @@ # Installables -Many `nix` subcommands operate on one or more *installables*. These are -command line arguments that represent something that can be built in -the Nix store. Here are the recognised types of installables: - -* **Flake output attributes**: `nixpkgs#hello` - - These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a - flake reference and *attrpath* is an optional attribute path. For - more information on flakes, see [the `nix flake` manual - page](./nix3-flake.md). Flake references are most commonly a flake - identifier in the flake registry (e.g. `nixpkgs`), or a raw path - (e.g. `/path/to/my-flake` or `.` or `../foo`), or a full URL - (e.g. `github:nixos/nixpkgs` or `path:.`) - - When the flake reference is a raw path (a path without any URL - scheme), it is interpreted as a `path:` or `git+file:` url in the following - way: - - - If the path is within a Git repository, then the url will be of the form - `git+file://[GIT_REPO_ROOT]?dir=[RELATIVE_FLAKE_DIR_PATH]` - where `GIT_REPO_ROOT` is the path to the root of the git repository, - and `RELATIVE_FLAKE_DIR_PATH` is the path (relative to the directory - root) of the closest parent of the given path that contains a `flake.nix` within - the git repository. - If no such directory exists, then Nix will error-out. - - Note that the search will only include files indexed by git. In particular, files - which are matched by `.gitignore` or have never been `git add`-ed will not be - available in the flake. If this is undesirable, specify `path:` explicitly; - - For example, if `/foo/bar` is a git repository with the following structure: - ``` - . - └── baz - ├── blah - │  └── file.txt - └── flake.nix - ``` +> **Warning** \ +> Installables are part of the unstable +> [`nix-command` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-nix-command), +> and subject to change without notice. + +Many `nix` subcommands operate on one or more *installables*. +These are command line arguments that represent something that can be realised in the Nix store. + +The following types of installable are supported by most commands: + +- [Flake output attribute](#flake-output-attribute) (experimental) +- [Store path](#store-path) +- [Nix file](#nix-file), optionally qualified by an attribute path +- [Nix expression](#nix-expression), optionally qualified by an attribute path + +For most commands, if no installable is specified, `.` as assumed. +That is, Nix will operate on the default flake output attribute of the flake in the current directory. + +### Flake output attribute + +> **Warning** \ +> Flake output attribute installables depend on both the +> [`flakes`](@docroot@/contributing/experimental-features.md#xp-feature-flakes) +> and +> [`nix-command`](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> experimental features, and subject to change without notice. + +Example: `nixpkgs#hello` + +These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a +[flake reference](./nix3-flake.md#flake-references) and *attrpath* is an optional attribute path. For +more information on flakes, see [the `nix flake` manual +page](./nix3-flake.md). Flake references are most commonly a flake +identifier in the flake registry (e.g. `nixpkgs`), or a raw path +(e.g. `/path/to/my-flake` or `.` or `../foo`), or a full URL +(e.g. `github:nixos/nixpkgs` or `path:.`) + +When the flake reference is a raw path (a path without any URL +scheme), it is interpreted as a `path:` or `git+file:` url in the following +way: + +- If the path is within a Git repository, then the url will be of the form + `git+file://[GIT_REPO_ROOT]?dir=[RELATIVE_FLAKE_DIR_PATH]` + where `GIT_REPO_ROOT` is the path to the root of the git repository, + and `RELATIVE_FLAKE_DIR_PATH` is the path (relative to the directory + root) of the closest parent of the given path that contains a `flake.nix` within + the git repository. + If no such directory exists, then Nix will error-out. + + Note that the search will only include files indexed by git. In particular, files + which are matched by `.gitignore` or have never been `git add`-ed will not be + available in the flake. If this is undesirable, specify `path:` explicitly; + + For example, if `/foo/bar` is a git repository with the following structure: + ``` + . + └── baz + ├── blah + │  └── file.txt + └── flake.nix + ``` Then `/foo/bar/baz/blah` will resolve to `git+file:///foo/bar?dir=baz` - - If the supplied path is not a git repository, then the url will have the form - `path:FLAKE_DIR_PATH` where `FLAKE_DIR_PATH` is the closest parent - of the supplied path that contains a `flake.nix` file (within the same file-system). - If no such directory exists, then Nix will error-out. - - For example, if `/foo/bar/flake.nix` exists, then `/foo/bar/baz/` will resolve to - `path:/foo/bar` - - If *attrpath* is omitted, Nix tries some default values; for most - subcommands, the default is `packages.`*system*`.default` - (e.g. `packages.x86_64-linux.default`), but some subcommands have - other defaults. If *attrpath* *is* specified, *attrpath* is - interpreted as relative to one or more prefixes; for most - subcommands, these are `packages.`*system*, - `legacyPackages.*system*` and the empty prefix. Thus, on - `x86_64-linux` `nix build nixpkgs#hello` will try to build the - attributes `packages.x86_64-linux.hello`, - `legacyPackages.x86_64-linux.hello` and `hello`. - -* **Store paths**: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` - - These are paths inside the Nix store, or symlinks that resolve to a - path in the Nix store. - -* **Store derivations**: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv` - - Store derivations are store paths with extension `.drv` and are a - low-level representation of a build-time dependency graph used - internally by Nix. By default, if you pass a store derivation to a - `nix` subcommand, it will operate on the *output paths* of the - derivation. For example, `nix path-info` prints information about - the output paths: +- If the supplied path is not a git repository, then the url will have the form + `path:FLAKE_DIR_PATH` where `FLAKE_DIR_PATH` is the closest parent + of the supplied path that contains a `flake.nix` file (within the same file-system). + If no such directory exists, then Nix will error-out. + + For example, if `/foo/bar/flake.nix` exists, then `/foo/bar/baz/` will resolve to + `path:/foo/bar` + +If *attrpath* is omitted, Nix tries some default values; for most +subcommands, the default is `packages.`*system*`.default` +(e.g. `packages.x86_64-linux.default`), but some subcommands have +other defaults. If *attrpath* *is* specified, *attrpath* is +interpreted as relative to one or more prefixes; for most +subcommands, these are `packages.`*system*, +`legacyPackages.*system*` and the empty prefix. Thus, on +`x86_64-linux` `nix build nixpkgs#hello` will try to build the +attributes `packages.x86_64-linux.hello`, +`legacyPackages.x86_64-linux.hello` and `hello`. + +### Store path + +Example: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` + +These are paths inside the Nix store, or symlinks that resolve to a path in the Nix store. + +A [store derivation] is also addressed by store path. + +Example: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv` + +If you want to refer to an output path of that store derivation, add the output name preceded by a caret (`^`). + +Example: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv^out` + +All outputs can be referred to at once with the special syntax `^*`. + +Example: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv^*` + +### Nix file + +Example: `--file /path/to/nixpkgs hello` + +When the option `-f` / `--file` *path* \[*attrpath*...\] is given, installables are interpreted as the value of the expression in the Nix file at *path*. +If attribute paths are provided, commands will operate on the corresponding values accessible at these paths. +The Nix expression in that file, or any selected attribute, must evaluate to a derivation. + +### Nix expression + +Example: `--expr 'import {}' hello` + +When the option `--expr` *expression* \[*attrpath*...\] is given, installables are interpreted as the value of the of the Nix expression. +If attribute paths are provided, commands will operate on the corresponding values accessible at these paths. +The Nix expression, or any selected attribute, must evaluate to a derivation. + +You may need to specify `--impure` if the expression references impure inputs (such as ``). + +## Derivation output selection + +Derivations can have multiple outputs, each corresponding to a +different store path. For instance, a package can have a `bin` output +that contains programs, and a `dev` output that provides development +artifacts like C/C++ header files. The outputs on which `nix` commands +operate are determined as follows: + +* You can explicitly specify the desired outputs using the syntax + *installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can + obtain the `dev` and `static` outputs of the `glibc` package: ```console - # nix path-info --json /nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv - [{"path":"/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10",…}] + # nix build 'nixpkgs#glibc^dev,static' + # ls ./result-dev/include/ ./result-static/lib/ + … ``` - If you want to operate on the store derivation itself, pass the - `--derivation` flag. + and likewise, using a store path to a "drv" file to specify the derivation: -* **Nix attributes**: `--file /path/to/nixpkgs hello` + ```console + # nix build '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^dev,static' + … + ``` - When the `-f` / `--file` *path* option is given, installables are - interpreted as attribute paths referencing a value returned by - evaluating the Nix file *path*. +* You can also specify that *all* outputs should be used using the + syntax *installable*`^*`. For example, the following shows the size + of all outputs of the `glibc` package in the binary cache: -* **Nix expressions**: `--expr '(import {}).hello.overrideDerivation (prev: { name = "my-hello"; })'`. + ```console + # nix path-info --closure-size --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*' + /nix/store/g02b1lpbddhymmcjb923kf0l7s9nww58-glibc-2.33-123 33208200 + /nix/store/851dp95qqiisjifi639r0zzg5l465ny4-glibc-2.33-123-bin 36142896 + /nix/store/kdgs3q6r7xdff1p7a9hnjr43xw2404z7-glibc-2.33-123-debug 155787312 + /nix/store/n4xa8h6pbmqmwnq0mmsz08l38abb06zc-glibc-2.33-123-static 42488328 + /nix/store/q6580lr01jpcsqs4r5arlh4ki2c1m9rv-glibc-2.33-123-dev 44200560 + ``` - When the `--expr` option is given, all installables are interpreted - as Nix expressions. You may need to specify `--impure` if the - expression references impure inputs (such as ``). + and likewise, using a store path to a "drv" file to specify the derivation: -For most commands, if no installable is specified, the default is `.`, -i.e. Nix will operate on the default flake output attribute of the -flake in the current directory. + ```console + # nix path-info --closure-size '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*' + … + ``` +* If you didn't specify the desired outputs, but the derivation has an + attribute `meta.outputsToInstall`, Nix will use those outputs. For + example, since the package `nixpkgs#libxml2` has this attribute: -# Nix stores + ```console + # nix eval 'nixpkgs#libxml2.meta.outputsToInstall' + [ "bin" "man" ] + ``` + + a command like `nix shell nixpkgs#libxml2` will provide only those + two outputs by default. -Most `nix` subcommands operate on a *Nix store*. + Note that a [store derivation] (given by its `.drv` file store path) doesn't have + any attributes like `meta`, and thus this case doesn't apply to it. + + [store derivation]: ../../glossary.md#gloss-store-derivation + +* Otherwise, Nix will use all outputs of the derivation. + +# Nix stores -TODO: list store types, options +Most `nix` subcommands operate on a *Nix store*. These are documented +in [`nix help-stores`](./nix3-help-stores.md). )"" diff -Nru nix-2.8.0/src/nix/path-from-hash-part.cc nix-2.16.1+dfsg/src/nix/path-from-hash-part.cc --- nix-2.8.0/src/nix/path-from-hash-part.cc 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/path-from-hash-part.cc 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,39 @@ +#include "command.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdPathFromHashPart : StoreCommand +{ + std::string hashPart; + + CmdPathFromHashPart() + { + expectArgs({ + .label = "hash-part", + .handler = {&hashPart}, + }); + } + + std::string description() override + { + return "get a store path from its hash part"; + } + + std::string doc() override + { + return + #include "path-from-hash-part.md" + ; + } + + void run(ref store) override + { + if (auto storePath = store->queryPathFromHashPart(hashPart)) + logger->cout(store->printStorePath(*storePath)); + else + throw Error("there is no store path corresponding to '%s'", hashPart); + } +}; + +static auto rCmdPathFromHashPart = registerCommand2({"store", "path-from-hash-part"}); diff -Nru nix-2.8.0/src/nix/path-from-hash-part.md nix-2.16.1+dfsg/src/nix/path-from-hash-part.md --- nix-2.8.0/src/nix/path-from-hash-part.md 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/path-from-hash-part.md 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,20 @@ +R""( + +# Examples + +* Return the full store path with the given hash part: + + ```console + # nix store path-from-hash-part --store https://cache.nixos.org/ 0i2jd68mp5g6h2sa5k9c85rb80sn8hi9 + /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 + ``` + +# Description + +Given the hash part of a store path (that is, the 32 characters +following `/nix/store/`), return the full store path. This is +primarily useful in the implementation of binary caches, where a +request for a `.narinfo` file only supplies the hash part +(e.g. `https://cache.nixos.org/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9.narinfo`). + +)"" diff -Nru nix-2.8.0/src/nix/path-info.cc nix-2.16.1+dfsg/src/nix/path-info.cc --- nix-2.8.0/src/nix/path-info.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/path-info.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,12 +1,13 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" -#include "json.hh" #include "common-args.hh" #include #include +#include + using namespace nix; struct CmdPathInfo : StorePathsCommand, MixJSON @@ -86,11 +87,10 @@ pathLen = std::max(pathLen, store->printStorePath(storePath).size()); if (json) { - JSONPlaceholder jsonRoot(std::cout); - store->pathInfoToJSON(jsonRoot, + std::cout << store->pathInfoToJSON( // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - true, showClosureSize, SRI, AllowInvalid); + true, showClosureSize, SRI, AllowInvalid).dump(); } else { diff -Nru nix-2.8.0/src/nix/path-info.md nix-2.16.1+dfsg/src/nix/path-info.md --- nix-2.8.0/src/nix/path-info.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/path-info.md 2023-06-06 10:55:23.000000000 +0000 @@ -13,7 +13,7 @@ closure, sorted by size: ```console - # nix path-info -rS /run/current-system | sort -nk2 + # nix path-info --recursive --closure-size /run/current-system | sort -nk2 /nix/store/hl5xwp9kdrd1zkm0idm3kkby9q66z404-empty 96 /nix/store/27324qvqhnxj3rncazmxc4mwy79kz8ha-nameservers 112 … @@ -25,7 +25,7 @@ readable sizes: ```console - # nix path-info -rsSh nixpkgs#rustc + # nix path-info --recursive --size --closure-size --human-readable nixpkgs#rustc /nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev 386.7K 69.1M /nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0 5.9M 37.4M … @@ -34,7 +34,7 @@ * Check the existence of a path in a binary cache: ```console - # nix path-info -r /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/ + # nix path-info --recursive /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/ path '/nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1' is not valid ``` @@ -57,7 +57,7 @@ size: ```console - # nix path-info --json --all -S \ + # nix path-info --json --all --closure-size \ | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])' [ …, @@ -68,7 +68,9 @@ ] ``` -* Print the path of the store derivation produced by `nixpkgs#hello`: +* Print the path of the [store derivation] produced by `nixpkgs#hello`: + + [store derivation]: ../../glossary.md#gloss-store-derivation ```console # nix path-info --derivation nixpkgs#hello @@ -78,7 +80,7 @@ # Description This command shows information about the store paths produced by -*installables*, or about all paths in the store if you pass `--all`. +[*installables*](./nix.md#installables), or about all paths in the store if you pass `--all`. By default, this command only prints the store paths. You can get additional information by passing flags such as `--closure-size`, diff -Nru nix-2.8.0/src/nix/ping-store.cc nix-2.16.1+dfsg/src/nix/ping-store.cc --- nix-2.8.0/src/nix/ping-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/ping-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,10 +1,13 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" +#include "finally.hh" + +#include using namespace nix; -struct CmdPingStore : StoreCommand +struct CmdPingStore : StoreCommand, MixJSON { std::string description() override { @@ -20,10 +23,26 @@ void run(ref store) override { - notice("Store URL: %s", store->getUri()); - store->connect(); - if (auto version = store->getVersion()) - notice("Version: %s", *version); + if (!json) { + notice("Store URL: %s", store->getUri()); + store->connect(); + if (auto version = store->getVersion()) + notice("Version: %s", *version); + if (auto trusted = store->isTrustedClient()) + notice("Trusted: %s", *trusted); + } else { + nlohmann::json res; + Finally printRes([&]() { + logger->cout("%s", res); + }); + + res["url"] = store->getUri(); + store->connect(); + if (auto version = store->getVersion()) + res["version"] = *version; + if (auto trusted = store->isTrustedClient()) + res["trusted"] = *trusted; + } } }; diff -Nru nix-2.8.0/src/nix/prefetch.cc nix-2.16.1+dfsg/src/nix/prefetch.cc --- nix-2.8.0/src/nix/prefetch.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/prefetch.cc 2023-06-06 10:55:23.000000000 +0000 @@ -27,18 +27,21 @@ Value vMirrors; // FIXME: use nixpkgs flake - state.eval(state.parseExprFromString("import ", "."), vMirrors); - state.forceAttrs(vMirrors, noPos); + state.eval(state.parseExprFromString( + "import ", + state.rootPath(CanonPath::root)), + vMirrors); + state.forceAttrs(vMirrors, noPos, "while evaluating the set of all mirrors"); auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); if (mirrorList == vMirrors.attrs->end()) throw Error("unknown mirror name '%s'", mirrorName); - state.forceList(*mirrorList->value, noPos); + state.forceList(*mirrorList->value, noPos, "while evaluating one mirror configuration"); if (mirrorList->value->listSize() < 1) throw Error("mirror URL '%s' did not expand to anything", url); - std::string mirror(state.forceString(*mirrorList->value->listElems()[0])); + std::string mirror(state.forceString(*mirrorList->value->listElems()[0], noPos, "while evaluating the first available mirror")); return mirror + (hasSuffix(mirror, "/") ? "" : "/") + s.substr(p + 1); } @@ -67,7 +70,13 @@ the store. */ if (expectedHash) { hashType = expectedHash->type; - storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { + .hash = { + .method = ingestionMethod, + .hash = *expectedHash, + }, + .references = {}, + }); if (store->isValidPath(*storePath)) hash = expectedHash; else @@ -118,7 +127,7 @@ auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); storePath = info.path; assert(info.ca); - hash = getContentAddressHash(*info.ca); + hash = info.ca->getHash(); } return {storePath.value(), hash.value()}; @@ -192,31 +201,35 @@ throw UsageError("you must specify a URL"); url = args[0]; } else { - Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); Value vRoot; - state->evalFile(path, vRoot); + state->evalFile( + resolveExprPath( + lookupFileArg(*state, args.empty() ? "." : args[0])), + vRoot); Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); - state->forceAttrs(v, noPos); + state->forceAttrs(v, noPos, "while evaluating the source attribute to prefetch"); /* Extract the URL. */ - auto & attr = v.attrs->need(state->symbols.create("urls")); - state->forceList(*attr.value, noPos); - if (attr.value->listSize() < 1) + auto * attr = v.attrs->get(state->symbols.create("urls")); + if (!attr) + throw Error("attribute 'urls' missing"); + state->forceList(*attr->value, noPos, "while evaluating the urls to prefetch"); + if (attr->value->listSize() < 1) throw Error("'urls' list is empty"); - url = state->forceString(*attr.value->listElems()[0]); + url = state->forceString(*attr->value->listElems()[0], noPos, "while evaluating the first url from the urls list"); /* Extract the hash mode. */ auto attr2 = v.attrs->get(state->symbols.create("outputHashMode")); if (!attr2) printInfo("warning: this does not look like a fetchurl call"); else - unpack = state->forceString(*attr2->value) == "recursive"; + unpack = state->forceString(*attr2->value, noPos, "while evaluating the outputHashMode of the source to prefetch") == "recursive"; /* Extract the name. */ if (!name) { auto attr3 = v.attrs->get(state->symbols.create("name")); if (!attr3) - name = state->forceString(*attr3->value); + name = state->forceString(*attr3->value, noPos, "while evaluating the name of the source to prefetch"); } } @@ -232,9 +245,9 @@ if (!printPath) printInfo("path is '%s'", store->printStorePath(storePath)); - std::cout << printHash16or32(hash) << std::endl; + logger->cout(printHash16or32(hash)); if (printPath) - std::cout << store->printStorePath(storePath) << std::endl; + logger->cout(store->printStorePath(storePath)); return 0; } diff -Nru nix-2.8.0/src/nix/print-dev-env.md nix-2.16.1+dfsg/src/nix/print-dev-env.md --- nix-2.8.0/src/nix/print-dev-env.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/print-dev-env.md 2023-06-06 10:55:23.000000000 +0000 @@ -40,7 +40,7 @@ This command prints a shell script that can be sourced by `bash` and that sets the variables and shell functions defined by the build -process of *installable*. This allows you to get a similar build +process of [*installable*](./nix.md#installables). This allows you to get a similar build environment in your current shell rather than in a subshell (as with `nix develop`). diff -Nru nix-2.8.0/src/nix/profile.cc nix-2.16.1+dfsg/src/nix/profile.cc --- nix-2.8.0/src/nix/profile.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/profile.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -22,33 +23,54 @@ // FIXME: record original attrpath. FlakeRef resolvedRef; std::string attrPath; - // FIXME: output names + ExtendedOutputsSpec outputs; bool operator < (const ProfileElementSource & other) const { return - std::pair(originalRef.to_string(), attrPath) < - std::pair(other.originalRef.to_string(), other.attrPath); + std::tuple(originalRef.to_string(), attrPath, outputs) < + std::tuple(other.originalRef.to_string(), other.attrPath, other.outputs); + } + + std::string to_string() const + { + return fmt("%s#%s%s", originalRef, attrPath, outputs.to_string()); } }; +const int defaultPriority = 5; + struct ProfileElement { StorePathSet storePaths; std::optional source; bool active = true; - // FIXME: priority + int priority = defaultPriority; - std::string describe() const + std::string identifier() const { if (source) - return fmt("%s#%s", source->originalRef, source->attrPath); + return source->to_string(); StringSet names; for (auto & path : storePaths) names.insert(DrvName(path.name()).name); return concatStringsSep(", ", names); } + /** + * Return a string representing an installable corresponding to the current + * element, either a flakeref or a plain store path + */ + std::set toInstallables(Store & store) + { + if (source) + return {source->to_string()}; + StringSet rawPaths; + for (auto & path : storePaths) + rawPaths.insert(store.printStorePath(path)); + return rawPaths; + } + std::string versions() const { StringSet versions; @@ -59,7 +81,7 @@ bool operator < (const ProfileElement & other) const { - return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); + return std::tuple(identifier(), storePaths) < std::tuple(other.identifier(), other.storePaths); } void updateStorePaths( @@ -67,7 +89,6 @@ ref store, const BuiltPaths & builtPaths) { - // FIXME: respect meta.outputsToInstall storePaths.clear(); for (auto & buildable : builtPaths) { std::visit(overloaded { @@ -99,7 +120,7 @@ auto version = json.value("version", 0); std::string sUrl; std::string sOriginalUrl; - switch(version){ + switch (version) { case 1: sUrl = "uri"; sOriginalUrl = "originalUri"; @@ -117,11 +138,15 @@ for (auto & p : e["storePaths"]) element.storePaths.insert(state.store->parseStorePath((std::string) p)); element.active = e["active"]; - if (e.value(sUrl,"") != "") { - element.source = ProfileElementSource{ + if(e.contains("priority")) { + element.priority = e["priority"]; + } + if (e.value(sUrl, "") != "") { + element.source = ProfileElementSource { parseFlakeRef(e[sOriginalUrl]), parseFlakeRef(e[sUrl]), - e["attrPath"] + e["attrPath"], + e["outputs"].get() }; } elements.emplace_back(std::move(element)); @@ -153,10 +178,12 @@ nlohmann::json obj; obj["storePaths"] = paths; obj["active"] = element.active; + obj["priority"] = element.priority; if (element.source) { obj["originalUrl"] = element.source->originalRef.to_string(); obj["url"] = element.source->resolvedRef.to_string(); obj["attrPath"] = element.source->attrPath; + obj["outputs"] = element.source->outputs; } array.push_back(obj); } @@ -176,7 +203,7 @@ for (auto & element : elements) { for (auto & path : element.storePaths) { if (element.active) - pkgs.emplace_back(store->printStorePath(path), true, 5); + pkgs.emplace_back(store->printStorePath(path), true, element.priority); references.insert(path); } } @@ -192,12 +219,22 @@ auto narHash = hashString(htSHA256, sink.s); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + *store, + "profile", + FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + .references = { + .others = std::move(references), + // profiles never refer to themselves + .self = false, + }, + }, narHash, }; - info.references = std::move(references); info.narSize = sink.s.size(); - info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; StringSource source(sink.s); store->addToStore(info, source); @@ -219,13 +256,13 @@ bool changes = false; while (i != prevElems.end() || j != curElems.end()) { - if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) { - std::cout << fmt("%s%s: ∅ -> %s\n", indent, j->describe(), j->versions()); + if (j != curElems.end() && (i == prevElems.end() || i->identifier() > j->identifier())) { + logger->cout("%s%s: ∅ -> %s", indent, j->identifier(), j->versions()); changes = true; ++j; } - else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) { - std::cout << fmt("%s%s: %s -> ∅\n", indent, i->describe(), i->versions()); + else if (i != prevElems.end() && (j == curElems.end() || i->identifier() < j->identifier())) { + logger->cout("%s%s: %s -> ∅", indent, i->identifier(), i->versions()); changes = true; ++i; } @@ -233,7 +270,7 @@ auto v1 = i->versions(); auto v2 = j->versions(); if (v1 != v2) { - std::cout << fmt("%s%s: %s -> %s\n", indent, i->describe(), v1, v2); + logger->cout("%s%s: %s -> %s", indent, i->identifier(), v1, v2); changes = true; } ++i; @@ -242,22 +279,46 @@ } if (!changes) - std::cout << fmt("%sNo changes.\n", indent); + logger->cout("%sNo changes.", indent); } }; -static std::map +static std::map>> builtPathsPerInstallable( - const std::vector, BuiltPath>> & builtPaths) + const std::vector, BuiltPathWithResult>> & builtPaths) { - std::map res; - for (auto & [installable, builtPath] : builtPaths) - res[installable.get()].push_back(builtPath); + std::map>> res; + for (auto & [installable, builtPath] : builtPaths) { + auto & r = res.insert({ + &*installable, + { + {}, + make_ref(), + } + }).first->second; + /* Note that there could be conflicting info + (e.g. meta.priority fields) if the installable returned + multiple derivations. So pick one arbitrarily. FIXME: + print a warning? */ + r.first.push_back(builtPath.path); + r.second = builtPath.info; + } return res; } struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { + std::optional priority; + + CmdProfileInstall() { + addFlag({ + .longName = "priority", + .description = "The priority of the package to install.", + .labels = {"priority"}, + .handler = {&priority}, + }); + }; + std::string description() override { return "install a package into a profile"; @@ -270,7 +331,7 @@ ; } - void run(ref store) override + void run(ref store, Installables && installables) override { ProfileManifest manifest(*getEvalState(), *profile); @@ -281,22 +342,93 @@ for (auto & installable : installables) { ProfileElement element; - if (auto installable2 = std::dynamic_pointer_cast(installable)) { - // FIXME: make build() return this? - auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); - element.source = ProfileElementSource{ - installable2->flakeRef, - resolvedRef, - attrPath, + auto iter = builtPaths.find(&*installable); + if (iter == builtPaths.end()) continue; + auto & [res, info] = iter->second; + + if (auto * info2 = dynamic_cast(&*info)) { + element.source = ProfileElementSource { + .originalRef = info2->flake.originalRef, + .resolvedRef = info2->flake.resolvedRef, + .attrPath = info2->value.attrPath, + .outputs = info2->value.extendedOutputsSpec, }; } - element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); + // If --priority was specified we want to override the + // priority of the installable. + element.priority = + priority + ? *priority + : ({ + auto * info2 = dynamic_cast(&*info); + info2 + ? info2->value.priority.value_or(defaultPriority) + : defaultPriority; + }); + + element.updateStorePaths(getEvalStore(), store, res); manifest.elements.push_back(std::move(element)); } - updateProfile(manifest.build(store)); + try { + updateProfile(manifest.build(store)); + } catch (BuildEnvFileConflictError & conflictError) { + // FIXME use C++20 std::ranges once macOS has it + // See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102 + auto findRefByFilePath = [&](Iterator begin, Iterator end) { + for (auto it = begin; it != end; it++) { + auto profileElement = *it; + for (auto & storePath : profileElement.storePaths) { + if (conflictError.fileA.starts_with(store->printStorePath(storePath))) { + return std::pair(conflictError.fileA, profileElement.toInstallables(*store)); + } + if (conflictError.fileB.starts_with(store->printStorePath(storePath))) { + return std::pair(conflictError.fileB, profileElement.toInstallables(*store)); + } + } + } + throw conflictError; + }; + // There are 2 conflicting files. We need to find out which one is from the already installed package and + // which one is the package that is the new package that is being installed. + // The first matching package is the one that was already installed (original). + auto [originalConflictingFilePath, originalConflictingRefs] = findRefByFilePath(manifest.elements.begin(), manifest.elements.end()); + // The last matching package is the one that was going to be installed (new). + auto [newConflictingFilePath, newConflictingRefs] = findRefByFilePath(manifest.elements.rbegin(), manifest.elements.rend()); + + throw Error( + "An existing package already provides the following file:\n" + "\n" + " %1%\n" + "\n" + "This is the conflicting file from the new package:\n" + "\n" + " %2%\n" + "\n" + "To remove the existing package:\n" + "\n" + " nix profile remove %3%\n" + "\n" + "The new package can also be installed next to the existing one by assigning a different priority.\n" + "The conflicting packages have a priority of %5%.\n" + "To prioritise the new package:\n" + "\n" + " nix profile install %4% --priority %6%\n" + "\n" + "To prioritise the existing package:\n" + "\n" + " nix profile install %4% --priority %7%\n", + originalConflictingFilePath, + newConflictingFilePath, + concatStringsSep(" ", originalConflictingRefs), + concatStringsSep(" ", newConflictingRefs), + conflictError.priority, + conflictError.priority - 1, + conflictError.priority + 1 + ); + } } }; @@ -378,7 +510,7 @@ if (!matches(*store, element, i, matchers)) { newManifest.elements.push_back(std::move(element)); } else { - notice("removing '%s'", element.describe()); + notice("removing '%s'", element.identifier()); } } @@ -423,7 +555,7 @@ auto matchers = getMatchers(store); - std::vector> installables; + Installables installables; std::vector indices; auto upgradedCount = 0; @@ -439,26 +571,33 @@ Activity act(*logger, lvlChatty, actUnknown, fmt("checking '%s' for updates", element.source->attrPath)); - auto installable = std::make_shared( + auto installable = make_ref( this, getEvalState(), FlakeRef(element.source->originalRef), "", + element.source->outputs, Strings{element.source->attrPath}, Strings{}, lockFlags); - auto [attrPath, resolvedRef, drv] = installable->toDerivation(); + auto derivedPaths = installable->toDerivedPaths(); + if (derivedPaths.empty()) continue; + auto * infop = dynamic_cast(&*derivedPaths[0].info); + // `InstallableFlake` should use `ExtraPathInfoFlake`. + assert(infop); + auto & info = *infop; - if (element.source->resolvedRef == resolvedRef) continue; + if (element.source->resolvedRef == info.flake.resolvedRef) continue; printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->resolvedRef, resolvedRef); + element.source->attrPath, element.source->resolvedRef, info.flake.resolvedRef); - element.source = ProfileElementSource{ - installable->flakeRef, - resolvedRef, - attrPath, + element.source = ProfileElementSource { + .originalRef = installable->flakeRef, + .resolvedRef = info.flake.resolvedRef, + .attrPath = info.value.attrPath, + .outputs = installable->extendedOutputsSpec, }; installables.push_back(installable); @@ -486,7 +625,10 @@ for (size_t i = 0; i < installables.size(); ++i) { auto & installable = installables.at(i); auto & element = manifest.elements[indices.at(i)]; - element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); + element.updateStorePaths( + getEvalStore(), + store, + builtPaths.find(&*installable)->second.first); } updateProfile(manifest.build(store)); @@ -514,8 +656,8 @@ for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); logger->cout("%d %s %s %s", i, - element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-", - element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-", + element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", + element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); } } @@ -544,9 +686,9 @@ for (auto & gen : gens) { if (prevGen) { - if (!first) std::cout << "\n"; + if (!first) logger->cout(""); first = false; - std::cout << fmt("Version %d -> %d:\n", prevGen->number, gen.number); + logger->cout("Version %d -> %d:", prevGen->number, gen.number); printClosureDiff(store, store->followLinksToStorePath(prevGen->path), store->followLinksToStorePath(gen.path), @@ -582,10 +724,10 @@ for (auto & gen : gens) { ProfileManifest manifest(*getEvalState(), gen.path); - if (!first) std::cout << "\n"; + if (!first) logger->cout(""); first = false; - std::cout << fmt("Version %s%d" ANSI_NORMAL " (%s)%s:\n", + logger->cout("Version %s%d" ANSI_NORMAL " (%s)%s:", gen.number == curGen ? ANSI_GREEN : ANSI_BOLD, gen.number, std::put_time(std::gmtime(&gen.creationTime), "%Y-%m-%d"), @@ -702,7 +844,6 @@ { if (!command) throw UsageError("'nix profile' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; diff -Nru nix-2.8.0/src/nix/profile-install.md nix-2.16.1+dfsg/src/nix/profile-install.md --- nix-2.8.0/src/nix/profile-install.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/profile-install.md 2023-06-06 10:55:23.000000000 +0000 @@ -20,8 +20,15 @@ # nix profile install nixpkgs/d73407e8e6002646acfdef0e39ace088bacc83da#hello ``` +* Install a specific output of a package: + + ```console + # nix profile install nixpkgs#bash^man + ``` + + # Description -This command adds *installables* to a Nix profile. +This command adds [*installables*](./nix.md#installables) to a Nix profile. )"" diff -Nru nix-2.8.0/src/nix/profile-list.md nix-2.16.1+dfsg/src/nix/profile-list.md --- nix-2.8.0/src/nix/profile-list.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/profile-list.md 2023-06-06 10:55:23.000000000 +0000 @@ -20,11 +20,11 @@ * An integer that can be used to unambiguously identify the package in invocations of `nix profile remove` and `nix profile upgrade`. -* The original ("mutable") flake reference and output attribute path +* The original ("unlocked") flake reference and output attribute path used at installation time. -* The immutable flake reference to which the mutable flake reference - was resolved. +* The locked flake reference to which the unlocked flake reference was + resolved. * The store path(s) of the package. diff -Nru nix-2.8.0/src/nix/profile.md nix-2.16.1+dfsg/src/nix/profile.md --- nix-2.8.0/src/nix/profile.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/profile.md 2023-06-06 10:55:23.000000000 +0000 @@ -7,101 +7,39 @@ independently from each other. Nix profiles are versioned, allowing them to be rolled back easily. -# Default profile +# Files -The default profile used by `nix profile` is `$HOME/.nix-profile`, -which, if it does not exist, is created as a symlink to -`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the -`root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise. +)"" -You can specify another profile location using `--profile` *path*. +#include "doc/files/profiles.md.gen.hh" -# Filesystem layout +R""( -Profiles are versioned as follows. When using profile *path*, *path* -is a symlink to *path*`-`*N*, where *N* is the current *version* of -the profile. In turn, *path*`-`*N* is a symlink to a path in the Nix -store. For example: +### Profile compatibility -```console -$ ls -l /nix/var/nix/profiles/per-user/alice/profile* -lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile -> profile-7-link -lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /nix/var/nix/profiles/per-user/alice/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile -lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /nix/var/nix/profiles/per-user/alice/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile -lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile -``` +> **Warning** +> +> Once you have used [`nix profile`] you can no longer use [`nix-env`] without first deleting `$XDG_STATE_HOME/nix/profiles/profile` -Each of these symlinks is a root for the Nix garbage collector. +[`nix-env`]: @docroot@/command-ref/nix-env.md +[`nix profile`]: @docroot@/command-ref/new-cli/nix3-profile.md -The contents of the store path corresponding to each version of the -profile is a tree of symlinks to the files of the installed packages, -e.g. +Once you installed a package with [`nix profile`], you get the following error message when using [`nix-env`]: ```console -$ ll -R /nix/var/nix/profiles/per-user/eelco/profile-7-link/ -/nix/var/nix/profiles/per-user/eelco/profile-7-link/: -total 20 -dr-xr-xr-x 2 root root 4096 Jan 1 1970 bin --r--r--r-- 2 root root 1402 Jan 1 1970 manifest.json -dr-xr-xr-x 4 root root 4096 Jan 1 1970 share - -/nix/var/nix/profiles/per-user/eelco/profile-7-link/bin: -total 20 -lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium -lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify -lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us - -/nix/var/nix/profiles/per-user/eelco/profile-7-link/share/applications: -total 12 -lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop -lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop -lrwxrwxrwx 3 root root 107 Jan 1 1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop - -… +$ nix-env -f '' -iA 'hello' +error: nix-env +profile '/home/alice/.local/state/nix/profiles/profile' is incompatible with 'nix-env'; please use 'nix profile' instead ``` -The file `manifest.json` records the provenance of the packages that -are installed in this version of the profile. It looks like this: +To migrate back to `nix-env` you can delete your current profile: -```json -{ - "version": 1, - "elements": [ - { - "active": true, - "attrPath": "legacyPackages.x86_64-linux.zoom-us", - "originalUrl": "flake:nixpkgs", - "storePaths": [ - "/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927" - ], - "uri": "github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a" - }, - … - ] -} -``` - -Each object in the array `elements` denotes an installed package and -has the following fields: +> **Warning** +> +> This will delete packages that have been installed before, so you may want to back up this information before running the command. -* `originalUrl`: The [flake reference](./nix3-flake.md) specified by - the user at the time of installation (e.g. `nixpkgs`). This is also - the flake reference that will be used by `nix profile upgrade`. - -* `uri`: The immutable flake reference to which `originalUrl` - resolved. - -* `attrPath`: The flake output attribute that provided this - package. Note that this is not necessarily the attribute that the - user specified, but the one resulting from applying the default - attribute paths and prefixes; for instance, `hello` might resolve to - `packages.x86_64-linux.hello` and the empty string to - `packages.x86_64-linux.default`. - -* `storePath`: The paths in the Nix store containing the package. - -* `active`: Whether the profile contains symlinks to the files of this - package. If set to false, the package is kept in the Nix store, but - is not "visible" in the profile's symlink tree. +```console + $ rm -rf "${XDG_STATE_HOME-$HOME/.local/state}/nix/profiles/profile" +``` )"" diff -Nru nix-2.8.0/src/nix/profile-upgrade.md nix-2.16.1+dfsg/src/nix/profile-upgrade.md --- nix-2.8.0/src/nix/profile-upgrade.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/profile-upgrade.md 2023-06-06 10:55:23.000000000 +0000 @@ -2,7 +2,7 @@ # Examples -* Upgrade all packages that were installed using a mutable flake +* Upgrade all packages that were installed using an unlocked flake reference: ```console @@ -32,9 +32,9 @@ > **Warning** > -> This only works if you used a *mutable* flake reference at +> This only works if you used an *unlocked* flake reference at > installation time, e.g. `nixpkgs#hello`. It does not work if you -> used an *immutable* flake reference +> used a *locked* flake reference > (e.g. `github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a#hello`), > since in that case the "latest version" is always the same. diff -Nru nix-2.8.0/src/nix/realisation.cc nix-2.16.1+dfsg/src/nix/realisation.cc --- nix-2.8.0/src/nix/realisation.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/realisation.cc 2023-06-06 10:55:23.000000000 +0000 @@ -21,7 +21,6 @@ { if (!command) throw UsageError("'nix realisation' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; @@ -46,7 +45,7 @@ void run(ref store, BuiltPaths && paths) override { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); RealisedPath::Set realisations; for (auto & builtPath : paths) { @@ -65,18 +64,16 @@ res.push_back(currentPath); } - std::cout << res.dump(); + logger->cout("%s", res); } else { for (auto & path : realisations) { if (auto realisation = std::get_if(&path.raw)) { - std::cout << - realisation->id.to_string() << " " << - store->printStorePath(realisation->outPath); + logger->cout("%s %s", + realisation->id.to_string(), + store->printStorePath(realisation->outPath)); } else - std::cout << store->printStorePath(path.path()); - - std::cout << std::endl; + logger->cout("%s", store->printStorePath(path.path())); } } } diff -Nru nix-2.8.0/src/nix/registry.cc nix-2.16.1+dfsg/src/nix/registry.cc --- nix-2.8.0/src/nix/registry.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/registry.cc 2023-06-06 10:55:23.000000000 +0000 @@ -183,14 +183,12 @@ void run(nix::ref store) override { - if (locked.empty()) { - locked = url; - } + if (locked.empty()) locked = url; auto registry = getRegistry(); auto ref = parseFlakeRef(url); - auto locked_ref = parseFlakeRef(locked); + auto lockedRef = parseFlakeRef(locked); registry->remove(ref.input); - auto [tree, resolved] = locked_ref.resolve(store).input.fetch(store); + auto [tree, resolved] = lockedRef.resolve(store).input.fetch(store); fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; registry->add(ref.input, resolved, extraAttrs); @@ -226,10 +224,9 @@ void run() override { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); if (!command) throw UsageError("'nix registry' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; diff -Nru nix-2.8.0/src/nix/registry.md nix-2.16.1+dfsg/src/nix/registry.md --- nix-2.8.0/src/nix/registry.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/registry.md 2023-06-06 10:55:23.000000000 +0000 @@ -29,7 +29,7 @@ can be specified using the NixOS option `nix.registry`. * The user registry `~/.config/nix/registry.json`. This registry can - be modified by commands such as `nix flake pin`. + be modified by commands such as `nix registry pin`. * Overrides specified on the command line using the option `--override-flake`. diff -Nru nix-2.8.0/src/nix/repl.cc nix-2.16.1+dfsg/src/nix/repl.cc --- nix-2.8.0/src/nix/repl.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/repl.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,890 +1,35 @@ -#include -#include -#include -#include - -#include - -#ifdef READLINE -#include -#include -#else -// editline < 1.15.2 don't wrap their API for C++ usage -// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). -// This results in linker errors due to to name-mangling of editline C symbols. -// For compatibility with these versions, we wrap the API here -// (wrapping multiple times on newer versions is no problem). -extern "C" { -#include -} -#endif - -#include "ansicolor.hh" -#include "shared.hh" #include "eval.hh" -#include "eval-inline.hh" -#include "attr-path.hh" -#include "store-api.hh" -#include "log-store.hh" -#include "common-eval-args.hh" -#include "get-drvs.hh" -#include "derivations.hh" #include "globals.hh" #include "command.hh" -#include "finally.hh" -#include "markdown.hh" - -#if HAVE_BOEHMGC -#define GC_INCLUDE_NEW -#include -#endif +#include "installable-value.hh" +#include "repl.hh" namespace nix { -struct NixRepl - #if HAVE_BOEHMGC - : gc - #endif -{ - std::string curDir; - std::unique_ptr state; - Bindings * autoArgs; - - Strings loadedFiles; - - const static int envSize = 32768; - StaticEnv staticEnv; - Env * env; - int displ; - StringSet varNames; - - const Path historyFile; - - NixRepl(const Strings & searchPath, nix::ref store); - ~NixRepl(); - void mainLoop(const std::vector & files); - StringSet completePrefix(const std::string & prefix); - bool getLine(std::string & input, const std::string &prompt); - StorePath getDerivationPath(Value & v); - bool processLine(std::string line); - void loadFile(const Path & path); - void loadFlake(const std::string & flakeRef); - void initEnv(); - void reloadFiles(); - void addAttrsToScope(Value & attrs); - void addVarToScope(const Symbol & name, Value & v); - Expr * parseString(std::string s); - void evalString(std::string s, Value & v); - - typedef std::set ValuesSeen; - std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth); - std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen); -}; - - -std::string removeWhitespace(std::string s) -{ - s = chomp(s); - size_t n = s.find_first_not_of(" \n\r\t"); - if (n != std::string::npos) s = std::string(s, n); - return s; -} - - -NixRepl::NixRepl(const Strings & searchPath, nix::ref store) - : state(std::make_unique(searchPath, store)) - , staticEnv(false, &state->staticBaseEnv) - , historyFile(getDataDir() + "/nix/repl-history") -{ - curDir = absPath("."); -} - - -NixRepl::~NixRepl() -{ - write_history(historyFile.c_str()); -} - -std::string runNix(Path program, const Strings & args, - const std::optional & input = {}) -{ - auto subprocessEnv = getEnv(); - subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - - auto res = runProgram(RunOptions { - .program = settings.nixBinDir+ "/" + program, - .args = args, - .environment = subprocessEnv, - .input = input, - }); - - if (!statusOk(res.first)) - throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); - - return res.second; -} - -static NixRepl * curRepl; // ugly - -static char * completionCallback(char * s, int *match) { - auto possible = curRepl->completePrefix(s); - if (possible.size() == 1) { - *match = 1; - auto *res = strdup(possible.begin()->c_str() + strlen(s)); - if (!res) throw Error("allocation failure"); - return res; - } else if (possible.size() > 1) { - auto checkAllHaveSameAt = [&](size_t pos) { - auto &first = *possible.begin(); - for (auto &p : possible) { - if (p.size() <= pos || p[pos] != first[pos]) - return false; - } - return true; - }; - size_t start = strlen(s); - size_t len = 0; - while (checkAllHaveSameAt(start + len)) ++len; - if (len > 0) { - *match = 1; - auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); - if (!res) throw Error("allocation failure"); - return res; - } - } - - *match = 0; - return nullptr; -} - -static int listPossibleCallback(char *s, char ***avp) { - auto possible = curRepl->completePrefix(s); - - if (possible.size() > (INT_MAX / sizeof(char*))) - throw Error("too many completions"); - - int ac = 0; - char **vp = nullptr; - - auto check = [&](auto *p) { - if (!p) { - if (vp) { - while (--ac >= 0) - free(vp[ac]); - free(vp); - } - throw Error("allocation failure"); - } - return p; - }; - - vp = check((char **)malloc(possible.size() * sizeof(char*))); - - for (auto & p : possible) - vp[ac++] = check(strdup(p.c_str())); - - *avp = vp; - - return ac; -} - -namespace { - // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. - volatile sig_atomic_t g_signal_received = 0; - - void sigintHandler(int signo) { - g_signal_received = signo; - } -} - -void NixRepl::mainLoop(const std::vector & files) -{ - std::string error = ANSI_RED "error:" ANSI_NORMAL " "; - notice("Welcome to Nix " + nixVersion + ". Type :? for help.\n"); - - for (auto & i : files) - loadedFiles.push_back(i); - - reloadFiles(); - if (!loadedFiles.empty()) notice(""); - - // Allow nix-repl specific settings in .inputrc - rl_readline_name = "nix-repl"; - createDirs(dirOf(historyFile)); -#ifndef READLINE - el_hist_size = 1000; -#endif - read_history(historyFile.c_str()); - curRepl = this; -#ifndef READLINE - rl_set_complete_func(completionCallback); - rl_set_list_possib_func(listPossibleCallback); -#endif - - std::string input; - - while (true) { - // When continuing input from previous lines, don't print a prompt, just align to the same - // number of chars as the prompt. - if (!getLine(input, input.empty() ? "nix-repl> " : " ")) - break; - - try { - if (!removeWhitespace(input).empty() && !processLine(input)) return; - } catch (ParseError & e) { - if (e.msg().find("unexpected end of file") != std::string::npos) { - // For parse errors on incomplete input, we continue waiting for the next line of - // input without clearing the input so far. - continue; - } else { - printMsg(lvlError, e.msg()); - } - } catch (Error & e) { - printMsg(lvlError, e.msg()); - } catch (Interrupted & e) { - printMsg(lvlError, e.msg()); - } - - // We handled the current input fully, so we should clear it - // and read brand new input. - input.clear(); - std::cout << std::endl; - } -} - - -bool NixRepl::getLine(std::string & input, const std::string & prompt) -{ - struct sigaction act, old; - sigset_t savedSignalMask, set; - - auto setupSignals = [&]() { - act.sa_handler = sigintHandler; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGINT, &act, &old)) - throw SysError("installing handler for SIGINT"); - - sigemptyset(&set); - sigaddset(&set, SIGINT); - if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) - throw SysError("unblocking SIGINT"); - }; - auto restoreSignals = [&]() { - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); - - if (sigaction(SIGINT, &old, 0)) - throw SysError("restoring handler for SIGINT"); - }; - - setupSignals(); - Finally resetTerminal([&]() { rl_deprep_terminal(); }); - char * s = readline(prompt.c_str()); - Finally doFree([&]() { free(s); }); - restoreSignals(); - - if (g_signal_received) { - g_signal_received = 0; - input.clear(); - return true; - } - - if (!s) - return false; - input += s; - input += '\n'; - return true; -} - - -StringSet NixRepl::completePrefix(const std::string & prefix) -{ - StringSet completions; - - size_t start = prefix.find_last_of(" \n\r\t(){}[]"); - std::string prev, cur; - if (start == std::string::npos) { - prev = ""; - cur = prefix; - } else { - prev = std::string(prefix, 0, start + 1); - cur = std::string(prefix, start + 1); - } - - size_t slash, dot; - - if ((slash = cur.rfind('/')) != std::string::npos) { - try { - auto dir = std::string(cur, 0, slash); - auto prefix2 = std::string(cur, slash + 1); - for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { - if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) - completions.insert(prev + dir + "/" + entry.name); - } - } catch (Error &) { - } - } else if ((dot = cur.rfind('.')) == std::string::npos) { - /* This is a variable name; look it up in the current scope. */ - StringSet::iterator i = varNames.lower_bound(cur); - while (i != varNames.end()) { - if (i->substr(0, cur.size()) != cur) break; - completions.insert(prev + *i); - i++; - } - } else { - try { - /* This is an expression that should evaluate to an - attribute set. Evaluate it to get the names of the - attributes. */ - auto expr = cur.substr(0, dot); - auto cur2 = cur.substr(dot + 1); - - Expr * e = parseString(expr); - Value v; - e->eval(*state, *env, v); - state->forceAttrs(v, noPos); - - for (auto & i : *v.attrs) { - std::string name = i.name; - if (name.substr(0, cur2.size()) != cur2) continue; - completions.insert(prev + expr + "." + name); - } - - } catch (ParseError & e) { - // Quietly ignore parse errors. - } catch (EvalError & e) { - // Quietly ignore evaluation errors. - } catch (UndefinedVarError & e) { - // Quietly ignore undefined variable errors. - } catch (BadURL & e) { - // Quietly ignore BadURL flake-related errors. - } - } - - return completions; -} - - -static bool isVarName(std::string_view s) -{ - if (s.size() == 0) return false; - char c = s[0]; - if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; - for (auto & i : s) - if (!((i >= 'a' && i <= 'z') || - (i >= 'A' && i <= 'Z') || - (i >= '0' && i <= '9') || - i == '_' || i == '-' || i == '\'')) - return false; - return true; -} - - -StorePath NixRepl::getDerivationPath(Value & v) { - auto drvInfo = getDerivation(*state, v, false); - if (!drvInfo) - throw Error("expression does not evaluate to a derivation, so I can't build it"); - auto drvPath = drvInfo->queryDrvPath(); - if (!drvPath) - throw Error("expression did not evaluate to a valid derivation (no 'drvPath' attribute)"); - if (!state->store->isValidPath(*drvPath)) - throw Error("expression evaluated to invalid derivation '%s'", state->store->printStorePath(*drvPath)); - return *drvPath; -} - - -bool NixRepl::processLine(std::string line) -{ - line = trim(line); - if (line == "") return true; - - _isInterrupted = false; - - std::string command, arg; - - if (line[0] == ':') { - size_t p = line.find_first_of(" \n\r\t"); - command = line.substr(0, p); - if (p != std::string::npos) arg = removeWhitespace(line.substr(p)); - } else { - arg = line; - } - - if (command == ":?" || command == ":help") { - // FIXME: convert to Markdown, include in the 'nix repl' manpage. - std::cout - << "The following commands are available:\n" - << "\n" - << " Evaluate and print expression\n" - << " = Bind expression to variable\n" - << " :a Add attributes from resulting set to scope\n" - << " :b Build derivation\n" - << " :e Open package or function in $EDITOR\n" - << " :i Build derivation, then install result into current profile\n" - << " :l Load Nix expression and add it to scope\n" - << " :lf Load Nix flake and add it to scope\n" - << " :p Evaluate and print expression recursively\n" - << " :q Exit nix-repl\n" - << " :r Reload all files\n" - << " :s Build dependencies of derivation, then start nix-shell\n" - << " :t Describe result of evaluation\n" - << " :u Build derivation, then start nix-shell\n" - << " :doc Show documentation of a builtin function\n" - << " :log Show logs for a derivation\n" - << " :st [bool] Enable, disable or toggle showing traces for errors\n"; - } - - else if (command == ":a" || command == ":add") { - Value v; - evalString(arg, v); - addAttrsToScope(v); - } - - else if (command == ":l" || command == ":load") { - state->resetFileCache(); - loadFile(arg); - } - - else if (command == ":lf" || command == ":load-flake") { - loadFlake(arg); - } - - else if (command == ":r" || command == ":reload") { - state->resetFileCache(); - reloadFiles(); - } - - else if (command == ":e" || command == ":edit") { - Value v; - evalString(arg, v); - - Pos pos; - - if (v.type() == nPath || v.type() == nString) { - PathSet context; - auto filename = state->coerceToString(noPos, v, context); - pos.file = state->symbols.create(*filename); - } else if (v.isLambda()) { - pos = v.lambda.fun->pos; - } else { - // assume it's a derivation - pos = findPackageFilename(*state, v, arg); - } - - // Open in EDITOR - auto args = editorFor(pos); - auto editor = args.front(); - args.pop_front(); - - // runProgram redirects stdout to a StringSink, - // using runProgram2 to allow editors to display their UI - runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); - - // Reload right after exiting the editor - state->resetFileCache(); - reloadFiles(); - } - - else if (command == ":t") { - Value v; - evalString(arg, v); - logger->cout(showType(v)); - } - - else if (command == ":u") { - Value v, f, result; - evalString(arg, v); - evalString("drv: (import {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f); - state->callFunction(f, v, result, Pos()); - - StorePath drvPath = getDerivationPath(result); - runNix("nix-shell", {state->store->printStorePath(drvPath)}); - } - - else if (command == ":b" || command == ":i" || command == ":s" || command == ":log") { - Value v; - evalString(arg, v); - StorePath drvPath = getDerivationPath(v); - Path drvPathRaw = state->store->printStorePath(drvPath); - - if (command == ":b") { - state->store->buildPaths({DerivedPath::Built{drvPath}}); - auto drv = state->store->readDerivation(drvPath); - logger->cout("\nThis derivation produced the following outputs:"); - for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) - logger->cout(" %s -> %s", outputName, state->store->printStorePath(outputPath)); - } else if (command == ":i") { - runNix("nix-env", {"-i", drvPathRaw}); - } else if (command == ":log") { - settings.readOnlyMode = true; - Finally roModeReset([&]() { - settings.readOnlyMode = false; - }); - auto subs = getDefaultSubstituters(); - - subs.push_front(state->store); - - bool foundLog = false; - RunPager pager; - for (auto & sub : subs) { - auto * logSubP = dynamic_cast(&*sub); - if (!logSubP) { - printInfo("Skipped '%s' which does not support retrieving build logs", sub->getUri()); - continue; - } - auto & logSub = *logSubP; - - auto log = logSub.getBuildLog(drvPath); - if (log) { - printInfo("got build log for '%s' from '%s'", drvPathRaw, logSub.getUri()); - logger->writeToStdout(*log); - foundLog = true; - break; - } - } - if (!foundLog) throw Error("build log of '%s' is not available", drvPathRaw); - } else { - runNix("nix-shell", {drvPathRaw}); - } - } - - else if (command == ":p" || command == ":print") { - Value v; - evalString(arg, v); - printValue(std::cout, v, 1000000000) << std::endl; - } - - else if (command == ":q" || command == ":quit") - return false; - - else if (command == ":doc") { - Value v; - evalString(arg, v); - if (auto doc = state->getDoc(v)) { - std::string markdown; - - if (!doc->args.empty() && doc->name) { - auto args = doc->args; - for (auto & arg : args) - arg = "*" + arg + "*"; - - markdown += - "**Synopsis:** `builtins." + (std::string) (*doc->name) + "` " - + concatStringsSep(" ", args) + "\n\n"; - } - - markdown += stripIndentation(doc->doc); - - logger->cout(trim(renderMarkdownToTerminal(markdown))); - } else - throw Error("value does not have documentation"); - } - - else if (command == ":st" || command == ":show-trace") { - if (arg == "false" || (arg == "" && loggerSettings.showTrace)) { - std::cout << "not showing error traces\n"; - loggerSettings.showTrace = false; - } else if (arg == "true" || (arg == "" && !loggerSettings.showTrace)) { - std::cout << "showing error traces\n"; - loggerSettings.showTrace = true; - } else { - throw Error("unexpected argument '%s' to %s", arg, command); - }; - } - - else if (command != "") - throw Error("unknown command '%1%'", command); - - else { - size_t p = line.find('='); - std::string name; - if (p != std::string::npos && - p < line.size() && - line[p + 1] != '=' && - isVarName(name = removeWhitespace(line.substr(0, p)))) - { - Expr * e = parseString(line.substr(p + 1)); - Value & v(*state->allocValue()); - v.mkThunk(env, e); - addVarToScope(state->symbols.create(name), v); - } else { - Value v; - evalString(line, v); - printValue(std::cout, v, 1) << std::endl; - } - } - - return true; -} - - -void NixRepl::loadFile(const Path & path) -{ - loadedFiles.remove(path); - loadedFiles.push_back(path); - Value v, v2; - state->evalFile(lookupFileArg(*state, path), v); - state->autoCallFunction(*autoArgs, v, v2); - addAttrsToScope(v2); -} - -void NixRepl::loadFlake(const std::string & flakeRefS) +struct CmdRepl : RawInstallablesCommand { - if (flakeRefS.empty()) - throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); - if (evalSettings.pureEval && !flakeRef.input.isLocked()) - throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); - - Value v; - - flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, - flake::LockFlags { - .updateLockFile = false, - .useRegistries = !evalSettings.pureEval, - .allowMutable = !evalSettings.pureEval, - }), - v); - addAttrsToScope(v); -} - - -void NixRepl::initEnv() -{ - env = &state->allocEnv(envSize); - env->up = &state->baseEnv; - displ = 0; - staticEnv.vars.clear(); - - varNames.clear(); - for (auto & i : state->staticBaseEnv.vars) - varNames.insert(i.first); -} - - -void NixRepl::reloadFiles() -{ - initEnv(); - - Strings old = loadedFiles; - loadedFiles.clear(); - - bool first = true; - for (auto & i : old) { - if (!first) notice(""); - first = false; - notice("Loading '%1%'...", i); - loadFile(i); - } -} - - -void NixRepl::addAttrsToScope(Value & attrs) -{ - state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }); - if (displ + attrs.attrs->size() >= envSize) - throw Error("environment full; cannot add more variables"); - - for (auto & i : *attrs.attrs) { - staticEnv.vars.emplace_back(i.name, displ); - env->values[displ++] = i.value; - varNames.insert((std::string) i.name); + CmdRepl() { + evalSettings.pureEval = false; } - staticEnv.sort(); - staticEnv.deduplicate(); - notice("Added %1% variables.", attrs.attrs->size()); -} - -void NixRepl::addVarToScope(const Symbol & name, Value & v) -{ - if (displ >= envSize) - throw Error("environment full; cannot add more variables"); - if (auto oldVar = staticEnv.find(name); oldVar != staticEnv.vars.end()) - staticEnv.vars.erase(oldVar); - staticEnv.vars.emplace_back(name, displ); - staticEnv.sort(); - env->values[displ++] = &v; - varNames.insert((std::string) name); -} - - -Expr * NixRepl::parseString(std::string s) -{ - Expr * e = state->parseExprFromString(std::move(s), curDir, staticEnv); - return e; -} - - -void NixRepl::evalString(std::string s, Value & v) -{ - Expr * e = parseString(s); - e->eval(*state, *env, v); - state->forceValue(v, [&]() { return v.determinePos(noPos); }); -} - - -std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth) -{ - ValuesSeen seen; - return printValue(str, v, maxDepth, seen); -} - - -std::ostream & printStringValue(std::ostream & str, const char * string) { - str << "\""; - for (const char * i = string; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else str << *i; - str << "\""; - return str; -} - - -// FIXME: lot of cut&paste from Nix's eval.cc. -std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen) -{ - str.flush(); - checkInterrupt(); - - state->forceValue(v, [&]() { return v.determinePos(noPos); }); - - switch (v.type()) { - - case nInt: - str << ANSI_CYAN << v.integer << ANSI_NORMAL; - break; - - case nBool: - str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL; - break; - - case nString: - str << ANSI_WARNING; - printStringValue(str, v.string.s); - str << ANSI_NORMAL; - break; - - case nPath: - str << ANSI_GREEN << v.path << ANSI_NORMAL; // !!! escaping? - break; - - case nNull: - str << ANSI_CYAN "null" ANSI_NORMAL; - break; - - case nAttrs: { - seen.insert(&v); - - bool isDrv = state->isDerivation(v); - - if (isDrv) { - str << "«derivation "; - Bindings::iterator i = v.attrs->find(state->sDrvPath); - PathSet context; - if (i != v.attrs->end()) - str << state->store->printStorePath(state->coerceToStorePath(*i->pos, *i->value, context)); - else - str << "???"; - str << "»"; - } - - else if (maxDepth > 0) { - str << "{ "; - - typedef std::map Sorted; - Sorted sorted; - for (auto & i : *v.attrs) - sorted[i.name] = i.value; - - for (auto & i : sorted) { - if (isVarName(i.first)) - str << i.first; - else - printStringValue(str, i.first.c_str()); - str << " = "; - if (seen.count(i.second)) - str << "«repeated»"; - else - try { - printValue(str, *i.second, maxDepth - 1, seen); - } catch (AssertionError & e) { - str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL; - } - str << "; "; - } - - str << "}"; - } else - str << "{ ... }"; - - break; + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; } - case nList: - seen.insert(&v); - - str << "[ "; - if (maxDepth > 0) - for (auto elem : v.listItems()) { - if (seen.count(elem)) - str << "«repeated»"; - else - try { - printValue(str, *elem, maxDepth - 1, seen); - } catch (AssertionError & e) { - str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL; - } - str << " "; - } - else - str << "... "; - str << "]"; - break; - - case nFunction: - if (v.isLambda()) { - std::ostringstream s; - s << v.lambda.fun->pos; - str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; - } else if (v.isPrimOp()) { - str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; - } else if (v.isPrimOpApp()) { - str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; - } else { - abort(); - } - break; + std::vector files; - case nFloat: - str << v.fpoint; - break; - - default: - str << ANSI_RED "«unknown»" ANSI_NORMAL; - break; + Strings getDefaultFlakeAttrPaths() override + { + return {""}; } - return str; -} - -struct CmdRepl : StoreCommand, MixEvalArgs -{ - std::vector files; - - CmdRepl() + bool forceImpureByDefault() override { - expectArgs({ - .label = "files", - .handler = {&files}, - .completer = completePath - }); + return true; } std::string description() override @@ -899,12 +44,56 @@ ; } - void run(ref store) override + void applyDefaultInstallables(std::vector & rawInstallables) override { - evalSettings.pureEval = false; - auto repl = std::make_unique(searchPath, openStore()); + if (!experimentalFeatureSettings.isEnabled(Xp::ReplFlake) && !(file) && rawInstallables.size() >= 1) { + warn("future versions of Nix will require using `--file` to load a file"); + if (rawInstallables.size() > 1) + warn("more than one input file is not currently supported"); + auto filePath = rawInstallables[0].data(); + file = std::optional(filePath); + rawInstallables.front() = rawInstallables.back(); + rawInstallables.pop_back(); + } + if (rawInstallables.empty() && (file.has_value() || expr.has_value())) { + rawInstallables.push_back("."); + } + } + + void run(ref store, std::vector && rawInstallables) override + { + auto state = getEvalState(); + auto getValues = [&]()->AbstractNixRepl::AnnotatedValues{ + auto installables = parseInstallables(store, rawInstallables); + AbstractNixRepl::AnnotatedValues values; + for (auto & installable_: installables){ + auto & installable = InstallableValue::require(*installable_); + auto what = installable.what(); + if (file){ + auto [val, pos] = installable.toValue(*state); + auto what = installable.what(); + state->forceValue(*val, pos); + auto autoArgs = getAutoArgs(*state); + auto valPost = state->allocValue(); + state->autoCallFunction(*autoArgs, *val, *valPost); + state->forceValue(*valPost, pos); + values.push_back( {valPost, what }); + } else { + auto [val, pos] = installable.toValue(*state); + values.push_back( {val, what} ); + } + } + return values; + }; + auto repl = AbstractNixRepl::create( + searchPath, + openStore(), + state, + getValues + ); repl->autoArgs = getAutoArgs(*repl->state); - repl->mainLoop(files); + repl->initEnv(); + repl->mainLoop(); } }; diff -Nru nix-2.8.0/src/nix/repl.md nix-2.16.1+dfsg/src/nix/repl.md --- nix-2.8.0/src/nix/repl.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/repl.md 2023-06-06 10:55:23.000000000 +0000 @@ -24,10 +24,34 @@ * Interact with Nixpkgs in the REPL: ```console - # nix repl '' + # nix repl --file example.nix + Loading Installable ''... + Added 3 variables. - Loading ''... - Added 12428 variables. + # nix repl --expr '{a={b=3;c=4;};}' + Loading Installable ''... + Added 1 variables. + + # nix repl --expr '{a={b=3;c=4;};}' a + Loading Installable ''... + Added 1 variables. + + # nix repl --extra-experimental-features 'flakes repl-flake' nixpkgs + Loading Installable 'flake:nixpkgs#'... + Added 5 variables. + + nix-repl> legacyPackages.x86_64-linux.emacs.name + "emacs-27.1" + + nix-repl> legacyPackages.x86_64-linux.emacs.name + "emacs-27.1" + + nix-repl> :q + + # nix repl --expr 'import {}' + + Loading Installable ''... + Added 12439 variables. nix-repl> emacs.name "emacs-27.1" diff -Nru nix-2.8.0/src/nix/run.cc nix-2.16.1+dfsg/src/nix/run.cc --- nix-2.8.0/src/nix/run.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/run.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,5 +1,5 @@ #include "run.hh" -#include "command.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -9,6 +9,7 @@ #include "fs-accessor.hh" #include "progress-bar.hh" #include "eval.hh" +#include "build/personality.hh" #if __linux__ #include @@ -24,7 +25,8 @@ void runProgramInStore(ref store, const std::string & program, - const Strings & args) + const Strings & args, + std::optional system) { stopProgressBar(); @@ -44,14 +46,17 @@ throw Error("store '%s' is not a local store so it does not support command execution", store->getUri()); if (store->storeDir != store2->getRealStoreDir()) { - Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; + Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), std::string(system.value_or("")), program }; for (auto & arg : args) helperArgs.push_back(arg); - execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); + execv(getSelfExe().value_or("nix").c_str(), stringsToCharPtrs(helperArgs).data()); throw SysError("could not execute chroot helper"); } + if (system) + setPersonality(*system); + execvp(program.c_str(), stringsToCharPtrs(args).data()); throw SysError("unable to execute '%s'", program); @@ -92,7 +97,7 @@ ; } - void run(ref store) override + void run(ref store, Installables && installables) override { auto outPaths = Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); @@ -132,7 +137,7 @@ static auto rCmdShell = registerCommand("shell"); -struct CmdRun : InstallableCommand +struct CmdRun : InstallableValueCommand { using InstallableCommand::run; @@ -178,7 +183,7 @@ return res; } - void run(ref store) override + void run(ref store, ref installable) override { auto state = getEvalState(); @@ -199,6 +204,7 @@ int p = 1; std::string storeDir = argv[p++]; std::string realStoreDir = argv[p++]; + std::string system = argv[p++]; std::string cmd = argv[p++]; Strings args; while (p < argc) @@ -262,6 +268,9 @@ writeFile("/proc/self/uid_map", fmt("%d %d %d", uid, uid, 1)); writeFile("/proc/self/gid_map", fmt("%d %d %d", gid, gid, 1)); + if (system != "") + setPersonality(system); + execvp(cmd.c_str(), stringsToCharPtrs(args).data()); throw SysError("unable to exec '%s'", cmd); diff -Nru nix-2.8.0/src/nix/run.hh nix-2.16.1+dfsg/src/nix/run.hh --- nix-2.8.0/src/nix/run.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/run.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" @@ -6,6 +7,7 @@ void runProgramInStore(ref store, const std::string & program, - const Strings & args); + const Strings & args, + std::optional system = std::nullopt); } diff -Nru nix-2.8.0/src/nix/run.md nix-2.16.1+dfsg/src/nix/run.md --- nix-2.8.0/src/nix/run.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/run.md 2023-06-06 10:55:23.000000000 +0000 @@ -35,7 +35,7 @@ # Description -`nix run` builds and runs *installable*, which must evaluate to an +`nix run` builds and runs [*installable*](./nix.md#installables), which must evaluate to an *app* or a regular Nix derivation. If *installable* evaluates to an *app* (see below), it executes the diff -Nru nix-2.8.0/src/nix/search.cc nix-2.16.1+dfsg/src/nix/search.cc --- nix-2.8.0/src/nix/search.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/search.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,33 +1,44 @@ -#include "command.hh" +#include "command-installable-value.hh" #include "globals.hh" #include "eval.hh" #include "eval-inline.hh" #include "names.hh" #include "get-drvs.hh" #include "common-args.hh" -#include "json.hh" #include "shared.hh" #include "eval-cache.hh" #include "attr-path.hh" -#include "fmt.hh" +#include "hilite.hh" #include #include +#include using namespace nix; +using json = nlohmann::json; std::string wrap(std::string prefix, std::string s) { - return prefix + s + ANSI_NORMAL; + return concatStrings(prefix, s, ANSI_NORMAL); } -struct CmdSearch : InstallableCommand, MixJSON +struct CmdSearch : InstallableValueCommand, MixJSON { std::vector res; + std::vector excludeRes; CmdSearch() { expectArgs("regex", &res); + addFlag(Flag { + .longName = "exclude", + .shortName = 'e', + .description = "Hide packages whose attribute path, name or description contain *regex*.", + .labels = {"regex"}, + .handler = {[this](std::string s) { + excludeRes.push_back(s); + }}, + }); } std::string description() override @@ -45,12 +56,12 @@ Strings getDefaultFlakeAttrPaths() override { return { - "packages." + settings.thisSystem.get() + ".", - "legacyPackages." + settings.thisSystem.get() + "." + "packages." + settings.thisSystem.get(), + "legacyPackages." + settings.thisSystem.get() }; } - void run(ref store) override + void run(ref store, ref installable) override { settings.readOnlyMode = true; evalSettings.enableImportFromDerivation.setDefault(false); @@ -62,14 +73,20 @@ res.push_back("^"); std::vector regexes; + std::vector excludeRegexes; regexes.reserve(res.size()); + excludeRegexes.reserve(excludeRes.size()); for (auto & re : res) regexes.push_back(std::regex(re, std::regex::extended | std::regex::icase)); + for (auto & re : excludeRes) + excludeRegexes.emplace_back(re, std::regex::extended | std::regex::icase); + auto state = getEvalState(); - auto jsonOut = json ? std::make_unique(std::cout) : nullptr; + std::optional jsonOut; + if (json) jsonOut = json::object(); uint64_t results = 0; @@ -77,13 +94,15 @@ visit = [&](eval_cache::AttrCursor & cursor, const std::vector & attrPath, bool initialRecurse) { + auto attrPathS = state->symbols.resolve(attrPath); + Activity act(*logger, lvlInfo, actUnknown, - fmt("evaluating '%s'", concatStringsSep(".", attrPath))); + fmt("evaluating '%s'", concatStringsSep(".", attrPathS))); try { auto recurse = [&]() { for (const auto & attr : cursor.getAttrs()) { - auto cursor2 = cursor.getAttr(attr); + auto cursor2 = cursor.getAttr(state->symbols[attr]); auto attrPath2(attrPath); attrPath2.push_back(attr); visit(*cursor2, attrPath2, false); @@ -91,19 +110,27 @@ }; if (cursor.isDerivation()) { - DrvName name(cursor.getAttr("name")->getString()); + DrvName name(cursor.getAttr(state->sName)->getString()); - auto aMeta = cursor.maybeGetAttr("meta"); - auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr; + auto aMeta = cursor.maybeGetAttr(state->sMeta); + auto aDescription = aMeta ? aMeta->maybeGetAttr(state->sDescription) : nullptr; auto description = aDescription ? aDescription->getString() : ""; std::replace(description.begin(), description.end(), '\n', ' '); - auto attrPath2 = concatStringsSep(".", attrPath); + auto attrPath2 = concatStringsSep(".", attrPathS); std::vector attrPathMatches; std::vector descriptionMatches; std::vector nameMatches; bool found = false; + for (auto & regex : excludeRegexes) { + if ( + std::regex_search(attrPath2, regex) + || std::regex_search(name.name, regex) + || std::regex_search(description, regex)) + return; + } + for (auto & regex : regexes) { found = false; auto addAll = [&found](std::sregex_iterator it, std::vector & vec) { @@ -126,41 +153,42 @@ { results++; if (json) { - auto jsonElem = jsonOut->object(attrPath2); - jsonElem.attr("pname", name.name); - jsonElem.attr("version", name.version); - jsonElem.attr("description", description); + (*jsonOut)[attrPath2] = { + {"pname", name.name}, + {"version", name.version}, + {"description", description}, + }; } else { - auto name2 = hiliteMatches(name.name, std::move(nameMatches), ANSI_GREEN, "\e[0;2m"); + auto name2 = hiliteMatches(name.name, nameMatches, ANSI_GREEN, "\e[0;2m"); if (results > 1) logger->cout(""); logger->cout( "* %s%s", - wrap("\e[0;1m", hiliteMatches(attrPath2, std::move(attrPathMatches), ANSI_GREEN, "\e[0;1m")), + wrap("\e[0;1m", hiliteMatches(attrPath2, attrPathMatches, ANSI_GREEN, "\e[0;1m")), name.version != "" ? " (" + name.version + ")" : ""); if (description != "") logger->cout( - " %s", hiliteMatches(description, std::move(descriptionMatches), ANSI_GREEN, ANSI_NORMAL)); + " %s", hiliteMatches(description, descriptionMatches, ANSI_GREEN, ANSI_NORMAL)); } } } else if ( attrPath.size() == 0 - || (attrPath[0] == "legacyPackages" && attrPath.size() <= 2) - || (attrPath[0] == "packages" && attrPath.size() <= 2)) + || (attrPathS[0] == "legacyPackages" && attrPath.size() <= 2) + || (attrPathS[0] == "packages" && attrPath.size() <= 2)) recurse(); else if (initialRecurse) recurse(); - else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) { + else if (attrPathS[0] == "legacyPackages" && attrPath.size() > 2) { auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations); if (attr && attr->getBool()) recurse(); } } catch (EvalError & e) { - if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages")) + if (!(attrPath.size() > 0 && attrPathS[0] == "legacyPackages")) throw; } }; @@ -168,6 +196,9 @@ for (auto & cursor : installable->getCursors(*state)) visit(*cursor, cursor->getAttrPath(), true); + if (json) + logger->cout("%s", *jsonOut); + if (!json && !results) throw Error("no results for the given search term(s)!"); } diff -Nru nix-2.8.0/src/nix/search.md nix-2.16.1+dfsg/src/nix/search.md --- nix-2.8.0/src/nix/search.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/search.md 2023-06-06 10:55:23.000000000 +0000 @@ -43,18 +43,29 @@ # nix search nixpkgs 'firefox|chromium' ``` -* Search for packages containing `git'`and either `frontend` or `gui`: +* Search for packages containing `git` and either `frontend` or `gui`: ```console # nix search nixpkgs git 'frontend|gui' ``` +* Search for packages containing `neovim` but hide ones containing either `gui` or `python`: + + ```console + # nix search nixpkgs neovim --exclude 'python|gui' + ``` + or + + ```console + # nix search nixpkgs neovim --exclude 'python' --exclude 'gui' + ``` + # Description -`nix search` searches *installable* (which must be evaluatable, e.g. a -flake) for packages whose name or description matches all of the +`nix search` searches [*installable*](./nix.md#installables) (which can be evaluated, that is, a +flake or Nix expression, but not a store path or store derivation path) for packages whose name or description matches all of the regular expressions *regex*. For each matching package, It prints the -full attribute name (from the root of the installable), the version +full attribute name (from the root of the [installable](./nix.md#installables)), the version and the `meta.description` field, highlighting the substrings that were matched by the regular expressions. If no regular expressions are specified, all packages are shown. diff -Nru nix-2.8.0/src/nix/shell.md nix-2.16.1+dfsg/src/nix/shell.md --- nix-2.8.0/src/nix/shell.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/shell.md 2023-06-06 10:55:23.000000000 +0000 @@ -19,20 +19,26 @@ * Run GNU Hello: ```console - # nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!' + # nix shell nixpkgs#hello --command hello --greeting 'Hi everybody!' Hi everybody! ``` +* Run multiple commands in a shell environment: + + ```console + # nix shell nixpkgs#gnumake --command sh --command "cd src && make" + ``` + * Run GNU Hello in a chroot store: ```console - # nix shell --store ~/my-nix nixpkgs#hello -c hello + # nix shell --store ~/my-nix nixpkgs#hello --command hello ``` * Start a shell providing GNU Hello in a chroot store: ```console - # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive -c bash + # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive --command bash ``` Note that it's necessary to specify `bash` explicitly because your @@ -42,7 +48,7 @@ # Description `nix shell` runs a command in an environment in which the `$PATH` variable -provides the specified *installables*. If no command is specified, it starts the +provides the specified [*installables*](./nix.md#installable). If no command is specified, it starts the default shell of your user account specified by `$SHELL`. )"" diff -Nru nix-2.8.0/src/nix/show-config.cc nix-2.16.1+dfsg/src/nix/show-config.cc --- nix-2.8.0/src/nix/show-config.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/show-config.cc 2023-06-06 10:55:23.000000000 +0000 @@ -9,15 +9,44 @@ struct CmdShowConfig : Command, MixJSON { + std::optional name; + + CmdShowConfig() { + expectArgs({ + .label = {"name"}, + .optional = true, + .handler = {&name}, + }); + } + std::string description() override { - return "show the Nix configuration"; + return "show the Nix configuration or the value of a specific setting"; } Category category() override { return catUtility; } void run() override { + if (name) { + if (json) { + throw UsageError("'--json' is not supported when specifying a setting name"); + } + + std::map settings; + globalConfig.getSettings(settings); + auto setting = settings.find(*name); + + if (setting == settings.end()) { + throw Error("could not find setting '%1%'", *name); + } else { + const auto & value = setting->second.value; + logger->cout("%s", value); + } + + return; + } + if (json) { // FIXME: use appropriate JSON types (bool, ints, etc). logger->cout("%s", globalConfig.toJSON().dump()); diff -Nru nix-2.8.0/src/nix/show-derivation.cc nix-2.16.1+dfsg/src/nix/show-derivation.cc --- nix-2.8.0/src/nix/show-derivation.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/show-derivation.cc 1970-01-01 00:00:00.000000000 +0000 @@ -1,125 +0,0 @@ -// FIXME: integrate this with nix path-info? -// FIXME: rename to 'nix store show-derivation' or 'nix debug show-derivation'? - -#include "command.hh" -#include "common-args.hh" -#include "store-api.hh" -#include "archive.hh" -#include "json.hh" -#include "derivations.hh" - -using namespace nix; - -struct CmdShowDerivation : InstallablesCommand -{ - bool recursive = false; - - CmdShowDerivation() - { - addFlag({ - .longName = "recursive", - .shortName = 'r', - .description = "Include the dependencies of the specified derivations.", - .handler = {&recursive, true} - }); - } - - std::string description() override - { - return "show the contents of a store derivation"; - } - - std::string doc() override - { - return - #include "show-derivation.md" - ; - } - - Category category() override { return catUtility; } - - void run(ref store) override - { - auto drvPaths = Installable::toDerivations(store, installables, true); - - if (recursive) { - StorePathSet closure; - store->computeFSClosure(drvPaths, closure); - drvPaths = std::move(closure); - } - - { - - JSONObject jsonRoot(std::cout, true); - - for (auto & drvPath : drvPaths) { - if (!drvPath.isDerivation()) continue; - - auto drvObj(jsonRoot.object(store->printStorePath(drvPath))); - - auto drv = store->readDerivation(drvPath); - - { - auto outputsObj(drvObj.object("outputs")); - for (auto & [_outputName, output] : drv.outputs) { - auto & outputName = _outputName; // work around clang bug - auto outputObj { outputsObj.object(outputName) }; - std::visit(overloaded { - [&](const DerivationOutput::InputAddressed & doi) { - outputObj.attr("path", store->printStorePath(doi.path)); - }, - [&](const DerivationOutput::CAFixed & dof) { - outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName))); - outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); - outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); - }, - [&](const DerivationOutput::CAFloating & dof) { - outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); - }, - [&](const DerivationOutput::Deferred &) {}, - [&](const DerivationOutput::Impure & doi) { - outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); - outputObj.attr("impure", true); - }, - }, output.raw()); - } - } - - { - auto inputsList(drvObj.list("inputSrcs")); - for (auto & input : drv.inputSrcs) - inputsList.elem(store->printStorePath(input)); - } - - { - auto inputDrvsObj(drvObj.object("inputDrvs")); - for (auto & input : drv.inputDrvs) { - auto inputList(inputDrvsObj.list(store->printStorePath(input.first))); - for (auto & outputId : input.second) - inputList.elem(outputId); - } - } - - drvObj.attr("system", drv.platform); - drvObj.attr("builder", drv.builder); - - { - auto argsList(drvObj.list("args")); - for (auto & arg : drv.args) - argsList.elem(arg); - } - - { - auto envObj(drvObj.object("env")); - for (auto & var : drv.env) - envObj.attr(var.first, var.second); - } - } - - } - - std::cout << "\n"; - } -}; - -static auto rCmdShowDerivation = registerCommand("show-derivation"); diff -Nru nix-2.8.0/src/nix/show-derivation.md nix-2.16.1+dfsg/src/nix/show-derivation.md --- nix-2.8.0/src/nix/show-derivation.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/show-derivation.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,103 +0,0 @@ -R""( - -# Examples - -* Show the store derivation that results from evaluating the Hello - package: - - ```console - # nix show-derivation nixpkgs#hello - { - "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { - … - } - } - ``` - -* Show the full derivation graph (if available) that produced your - NixOS system: - - ```console - # nix show-derivation -r /run/current-system - ``` - -* Print all files fetched using `fetchurl` by Firefox's dependency - graph: - - ```console - # nix show-derivation -r nixpkgs#firefox \ - | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ - | uniq | sort - ``` - - Note that `.outputs.out.hash` selects *fixed-output derivations* - (derivations that produce output with a specified content hash), - while `.env.urls` selects derivations with a `urls` attribute. - -# Description - -This command prints on standard output a JSON representation of the -store derivations to which *installables* evaluate. Store derivations -are used internally by Nix. They are store paths with extension `.drv` -that represent the build-time dependency graph to which a Nix -expression evaluates. - -By default, this command only shows top-level derivations, but with -`--recursive`, it also shows their dependencies. - -The JSON output is a JSON object whose keys are the store paths of the -derivations, and whose values are a JSON object with the following -fields: - -* `outputs`: Information about the output paths of the - derivation. This is a JSON object with one member per output, where - the key is the output name and the value is a JSON object with these - fields: - - * `path`: The output path. - * `hashAlgo`: For fixed-output derivations, the hashing algorithm - (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a - NAR hash rather than a flat file hash. - * `hash`: For fixed-output derivations, the expected content hash in - base-16. - - Example: - - ```json - "outputs": { - "out": { - "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", - "hashAlgo": "r:sha256", - "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" - } - } - ``` - -* `inputSrcs`: A list of store paths on which this derivation depends. - -* `inputDrvs`: A JSON object specifying the derivations on which this - derivation depends, and what outputs of those derivations. For - example, - - ```json - "inputDrvs": { - "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], - "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] - } - ``` - - specifies that this derivation depends on the `dev` output of - `curl`, and the `out` output of `unzip`. - -* `system`: The system type on which this derivation is to be built - (e.g. `x86_64-linux`). - -* `builder`: The absolute path of the program to be executed to run - the build. Typically this is the `bash` shell - (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). - -* `args`: The command-line arguments passed to the `builder`. - -* `env`: The environment passed to the `builder`. - -)"" diff -Nru nix-2.8.0/src/nix/sigs.cc nix-2.16.1+dfsg/src/nix/sigs.cc --- nix-2.8.0/src/nix/sigs.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/sigs.cc 2023-06-06 10:55:23.000000000 +0000 @@ -45,7 +45,7 @@ //logger->setExpected(doneLabel, storePaths.size()); auto doPath = [&](const Path & storePathS) { - //Activity act(*logger, lvlInfo, format("getting signatures for '%s'") % storePath); + //Activity act(*logger, lvlInfo, "getting signatures for '%s'", storePath); checkInterrupt(); @@ -173,7 +173,7 @@ if (!keyName) throw UsageError("required argument '--key-name' is missing"); - std::cout << SecretKey::generate(*keyName).to_string(); + writeFull(STDOUT_FILENO, SecretKey::generate(*keyName).to_string()); } }; @@ -194,7 +194,7 @@ void run() override { SecretKey secretKey(drainFD(STDIN_FILENO)); - std::cout << secretKey.toPublicKey().to_string(); + writeFull(STDOUT_FILENO, secretKey.toPublicKey().to_string()); } }; @@ -219,7 +219,6 @@ { if (!command) throw UsageError("'nix key' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; diff -Nru nix-2.8.0/src/nix/store.cc nix-2.16.1+dfsg/src/nix/store.cc --- nix-2.8.0/src/nix/store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -18,7 +18,6 @@ { if (!command) throw UsageError("'nix store' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; diff -Nru nix-2.8.0/src/nix/store-copy-log.cc nix-2.16.1+dfsg/src/nix/store-copy-log.cc --- nix-2.8.0/src/nix/store-copy-log.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/store-copy-log.cc 2023-06-06 10:55:23.000000000 +0000 @@ -24,22 +24,14 @@ ; } - Category category() override { return catUtility; } - - void run(ref srcStore) override + void run(ref srcStore, Installables && installables) override { auto & srcLogStore = require(*srcStore); auto dstStore = getDstStore(); auto & dstLogStore = require(*dstStore); - StorePathSet drvPaths; - - for (auto & i : installables) - for (auto & drvPath : i->toDrvPaths(getEvalStore())) - drvPaths.insert(drvPath); - - for (auto & drvPath : drvPaths) { + for (auto & drvPath : Installable::toDerivations(getEvalStore(), installables, true)) { if (auto log = srcLogStore.getBuildLog(drvPath)) dstLogStore.addBuildLog(drvPath, *log); else diff -Nru nix-2.8.0/src/nix/store-copy-log.md nix-2.16.1+dfsg/src/nix/store-copy-log.md --- nix-2.8.0/src/nix/store-copy-log.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/store-copy-log.md 2023-06-06 10:55:23.000000000 +0000 @@ -18,7 +18,9 @@ (The flag `--substituters ''` avoids querying `https://cache.nixos.org` for the log.) -* To copy the log for a specific store derivation via SSH: +* To copy the log for a specific [store derivation] via SSH: + + [store derivation]: ../../glossary.md#gloss-store-derivation ```console # nix store copy-log --to ssh-ng://machine /nix/store/ilgm50plpmcgjhcp33z6n4qbnpqfhxym-glibc-2.33-59.drv diff -Nru nix-2.8.0/src/nix/store-delete.cc nix-2.16.1+dfsg/src/nix/store-delete.cc --- nix-2.8.0/src/nix/store-delete.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/store-delete.cc 2023-06-06 10:55:23.000000000 +0000 @@ -32,7 +32,7 @@ ; } - void run(ref store, std::vector && storePaths) override + void run(ref store, StorePaths && storePaths) override { auto & gcStore = require(*store); diff -Nru nix-2.8.0/src/nix/store-delete.md nix-2.16.1+dfsg/src/nix/store-delete.md --- nix-2.8.0/src/nix/store-delete.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/store-delete.md 2023-06-06 10:55:23.000000000 +0000 @@ -10,7 +10,7 @@ # Description -This command deletes the store paths specified by *installables*. , +This command deletes the store paths specified by [*installables*](./nix.md#installables), but only if it is safe to do so; that is, when the path is not reachable from a root of the garbage collector. This means that you can only delete paths that would also be deleted by `nix store diff -Nru nix-2.8.0/src/nix/store-dump-path.md nix-2.16.1+dfsg/src/nix/store-dump-path.md --- nix-2.8.0/src/nix/store-dump-path.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/store-dump-path.md 2023-06-06 10:55:23.000000000 +0000 @@ -18,6 +18,6 @@ # Description This command generates a NAR file containing the serialisation of the -store path *installable*. The NAR is written to standard output. +store path [*installable*](./nix.md#installables). The NAR is written to standard output. )"" diff -Nru nix-2.8.0/src/nix/store-ls.md nix-2.16.1+dfsg/src/nix/store-ls.md --- nix-2.8.0/src/nix/store-ls.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/store-ls.md 2023-06-06 10:55:23.000000000 +0000 @@ -5,7 +5,7 @@ * To list the contents of a store path in a binary cache: ```console - # nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 + # nix store ls --store https://cache.nixos.org/ --long --recursive /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 dr-xr-xr-x 0 ./bin -r-xr-xr-x 38184 ./bin/hello dr-xr-xr-x 0 ./share @@ -15,7 +15,7 @@ * To show information about a specific file in a binary cache: ```console - # nix store ls --store https://cache.nixos.org/ -l /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello + # nix store ls --store https://cache.nixos.org/ --long /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello -r-xr-xr-x 38184 hello ``` diff -Nru nix-2.8.0/src/nix/store-repair.cc nix-2.16.1+dfsg/src/nix/store-repair.cc --- nix-2.8.0/src/nix/store-repair.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/store-repair.cc 2023-06-06 10:55:23.000000000 +0000 @@ -17,7 +17,7 @@ ; } - void run(ref store, std::vector && storePaths) override + void run(ref store, StorePaths && storePaths) override { for (auto & path : storePaths) store->repairPath(path); diff -Nru nix-2.8.0/src/nix/store-repair.md nix-2.16.1+dfsg/src/nix/store-repair.md --- nix-2.8.0/src/nix/store-repair.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/store-repair.md 2023-06-06 10:55:23.000000000 +0000 @@ -17,7 +17,7 @@ # Description This command attempts to "repair" the store paths specified by -*installables* by redownloading them using the available +[*installables*](./nix.md#installables) by redownloading them using the available substituters. If no substitutes are available, then repair is not possible. diff -Nru nix-2.8.0/src/nix/upgrade-nix.cc nix-2.16.1+dfsg/src/nix/upgrade-nix.cc --- nix-2.8.0/src/nix/upgrade-nix.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/upgrade-nix.cc 2023-06-06 10:55:23.000000000 +0000 @@ -32,9 +32,17 @@ }); } + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + std::string description() override { - return "upgrade Nix to the latest stable version"; + return "upgrade Nix to the stable version declared in Nixpkgs"; } std::string doc() override @@ -140,11 +148,11 @@ auto state = std::make_unique(Strings(), store); auto v = state->allocValue(); - state->eval(state->parseExprFromString(res.data, "/no-such-path"), *v); + state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v).first; - return store->parseStorePath(state->forceString(*v2)); + return store->parseStorePath(state->forceString(*v2, noPos, "while evaluating the path tho latest nix version")); } }; diff -Nru nix-2.8.0/src/nix/upgrade-nix.md nix-2.16.1+dfsg/src/nix/upgrade-nix.md --- nix-2.8.0/src/nix/upgrade-nix.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/upgrade-nix.md 2023-06-06 10:55:23.000000000 +0000 @@ -2,7 +2,7 @@ # Examples -* Upgrade Nix to the latest stable version: +* Upgrade Nix to the stable version declared in Nixpkgs: ```console # nix upgrade-nix @@ -11,13 +11,16 @@ * Upgrade Nix in a specific profile: ```console - # nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile + # nix upgrade-nix --profile ~alice/.local/state/nix/profiles/profile ``` # Description -This command upgrades Nix to the latest version. By default, it -locates the directory containing the `nix` binary in the `$PATH` +This command upgrades Nix to the stable version declared in Nixpkgs. +This stable version is defined in [nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) +and updated manually. It may not always be the latest tagged release. + +By default, it locates the directory containing the `nix` binary in the `$PATH` environment variable. If that directory is a Nix profile, it will upgrade the `nix` package in that profile to the latest stable binary release. diff -Nru nix-2.8.0/src/nix/verify.cc nix-2.16.1+dfsg/src/nix/verify.cc --- nix-2.8.0/src/nix/verify.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/verify.cc 2023-06-06 10:55:23.000000000 +0000 @@ -41,7 +41,7 @@ addFlag({ .longName = "sigs-needed", .shortName = 'n', - .description = "Require that each path has at least *n* valid signatures.", + .description = "Require that each path is signed by at least *n* different keys.", .labels = {"n"}, .handler = {&sigsNeeded} }); @@ -81,14 +81,14 @@ ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -173,7 +173,7 @@ }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); diff -Nru nix-2.8.0/src/nix/verify.md nix-2.16.1+dfsg/src/nix/verify.md --- nix-2.8.0/src/nix/verify.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/verify.md 2023-06-06 10:55:23.000000000 +0000 @@ -12,7 +12,7 @@ signatures: ```console - # nix store verify -r -n2 --no-contents $(type -p firefox) + # nix store verify --recursive --sigs-needed 2 --no-contents $(type -p firefox) ``` * Verify a store path in the binary cache `https://cache.nixos.org/`: @@ -24,7 +24,7 @@ # Description -This command verifies the integrity of the store paths *installables*, +This command verifies the integrity of the store paths [*installables*](./nix.md#installables), or, if `--all` is given, the entire Nix store. For each path, it checks that diff -Nru nix-2.8.0/src/nix/why-depends.cc nix-2.16.1+dfsg/src/nix/why-depends.cc --- nix-2.8.0/src/nix/why-depends.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix/why-depends.cc 2023-06-06 10:55:23.000000000 +0000 @@ -27,7 +27,7 @@ return res; } -struct CmdWhyDepends : SourceExprCommand +struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions { std::string _package, _dependency; bool all = false; @@ -83,20 +83,37 @@ { auto package = parseInstallable(store, _package); auto packagePath = Installable::toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, package); + + /* We don't need to build `dependency`. We try to get the store + * path if it's already known, and if not, then it's not a dependency. + * + * Why? If `package` does depends on `dependency`, then getting the + * store path of `package` above necessitated having the store path + * of `dependency`. The contrapositive is, if the store path of + * `dependency` is not already known at this point (i.e. it's a CA + * derivation which hasn't been built), then `package` did not need it + * to build. + */ auto dependency = parseInstallable(store, _dependency); - auto dependencyPath = Installable::toStorePath(getEvalStore(), store, Realise::Derivation, operateOn, dependency); - auto dependencyPathHash = dependencyPath.hashPart(); + auto optDependencyPath = [&]() -> std::optional { + try { + return {Installable::toStorePath(getEvalStore(), store, Realise::Derivation, operateOn, dependency)}; + } catch (MissingRealisation &) { + return std::nullopt; + } + }(); StorePathSet closure; store->computeFSClosure({packagePath}, closure, false, false); - if (!closure.count(dependencyPath)) { - printError("'%s' does not depend on '%s'", - store->printStorePath(packagePath), - store->printStorePath(dependencyPath)); + if (!optDependencyPath.has_value() || !closure.count(*optDependencyPath)) { + printError("'%s' does not depend on '%s'", package->what(), dependency->what()); return; } + auto dependencyPath = *optDependencyPath; + auto dependencyPathHash = dependencyPath.hashPart(); + stopProgressBar(); // FIXME auto accessor = store->getFSAccessor(); diff -Nru nix-2.8.0/src/nix-build/nix-build.cc nix-2.16.1+dfsg/src/nix-build/nix-build.cc --- nix-2.8.0/src/nix-build/nix-build.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix-build/nix-build.cc 2023-06-06 10:55:23.000000000 +0000 @@ -84,8 +84,6 @@ auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO); Strings attrPaths; Strings left; - RepairFlag repair = NoRepair; - Path gcRoot; BuildMode buildMode = bmNormal; bool readStdin = false; @@ -167,17 +165,9 @@ else if (*arg == "--out-link" || *arg == "-o") outLink = getArg(*arg, arg, end); - else if (*arg == "--add-root") - gcRoot = getArg(*arg, arg, end); - else if (*arg == "--dry-run") dryRun = true; - else if (*arg == "--repair") { - repair = Repair; - buildMode = bmRepair; - } - else if (*arg == "--run-env") // obsolete runEnv = true; @@ -223,9 +213,9 @@ // read the shebang to understand which packages to read from. Since // this is handled via nix-shell -p, we wrap our ruby script execution // in ruby -e 'load' which ignores the shebangs. - envCommand = (format("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); + envCommand = fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, shellEscape(script), joined.str()); } else { - envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); + envCommand = fmt("exec %1% %2% %3% %4%", execArgs, interpreter, shellEscape(script), joined.str()); } } @@ -253,15 +243,17 @@ auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; auto state = std::make_unique(myArgs.searchPath, evalStore, store); - state->repair = repair; + state->repair = myArgs.repair; + if (myArgs.repair) buildMode = bmRepair; auto autoArgs = myArgs.getAutoArgs(*state); + auto autoArgsWithInNixShell = autoArgs; if (runEnv) { - auto newArgs = state->buildBindings(autoArgs->size() + 1); + auto newArgs = state->buildBindings(autoArgsWithInNixShell->size() + 1); newArgs.alloc("inNixShell").mkBool(true); for (auto & i : *autoArgs) newArgs.insert(i); - autoArgs = newArgs.finish(); + autoArgsWithInNixShell = newArgs.finish(); } if (packages) { @@ -292,7 +284,7 @@ else for (auto i : left) { if (fromArgs) - exprs.push_back(state->parseExprFromString(std::move(i), absPath("."))); + exprs.push_back(state->parseExprFromString(std::move(i), state->rootPath(CanonPath::fromCwd()))); else { auto absolute = i; try { @@ -316,10 +308,39 @@ Value vRoot; state->eval(e, vRoot); + std::function takesNixShellAttr; + takesNixShellAttr = [&](const Value & v) { + if (!runEnv) { + return false; + } + bool add = false; + if (v.type() == nFunction && v.lambda.fun->hasFormals()) { + for (auto & i : v.lambda.fun->formals->formals) { + if (state->symbols[i.name] == "inNixShell") { + add = true; + break; + } + } + } + return add; + }; + for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot).first); + Value & v(*findAlongAttrPath( + *state, + i, + takesNixShellAttr(vRoot) ? *autoArgsWithInNixShell : *autoArgs, + vRoot + ).first); state->forceValue(v, [&]() { return v.determinePos(noPos); }); - getDerivations(*state, v, "", *autoArgs, drvs, false); + getDerivations( + *state, + v, + "", + takesNixShellAttr(v) ? *autoArgsWithInNixShell : *autoArgs, + drvs, + false + ); } } @@ -359,7 +380,9 @@ if (!shell) { try { - auto expr = state->parseExprFromString("(import {}).bashInteractive", absPath(".")); + auto expr = state->parseExprFromString( + "(import {}).bashInteractive", + state->rootPath(CanonPath::fromCwd())); Value v; state->eval(expr, v); @@ -371,7 +394,7 @@ auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { .drvPath = bashDrv, - .outputs = {}, + .outputs = OutputsSpec::Names {"out"}, }); pathsToCopy.insert(bashDrv); shellDrv = bashDrv; @@ -395,7 +418,7 @@ { pathsToBuild.push_back(DerivedPath::Built { .drvPath = inputDrv, - .outputs = inputOutputs + .outputs = OutputsSpec::Names { inputOutputs }, }); pathsToCopy.insert(inputDrv); } @@ -414,7 +437,7 @@ shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash"; } - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto resolvedDrv = drv.tryResolve(*store); assert(resolvedDrv && "Successfully resolved the derivation"); drv = *resolvedDrv; @@ -440,7 +463,7 @@ env["NIX_STORE"] = store->storeDir; env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); - auto passAsFile = tokenizeString(get(drv.env, "passAsFile").value_or("")); + auto passAsFile = tokenizeString(getOr(drv.env, "passAsFile", "")); bool keepTmp = false; int fileNr = 0; @@ -510,7 +533,9 @@ "SHELL=%5%; " "BASH=%5%; " "set +e; " - R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" + R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && )s" + + (getuid() == 0 ? R"s(PS1='\n\[\033[1;31m\][nix-shell:\w]\$\[\033[0m\] '; )s" + : R"s(PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s") + "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " @@ -565,7 +590,7 @@ if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back(DerivedPath::Built{drvPath, {outputName}}); + pathsToBuild.push_back(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}}); pathsToBuildOrdered.push_back({drvPath, {outputName}}); drvsToCopy.insert(drvPath); diff -Nru nix-2.8.0/src/nix-channel/nix-channel.cc nix-2.16.1+dfsg/src/nix-channel/nix-channel.cc --- nix-2.8.0/src/nix-channel/nix-channel.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix-channel/nix-channel.cc 2023-06-06 10:55:23.000000000 +0000 @@ -1,9 +1,11 @@ +#include "profiles.hh" #include "shared.hh" #include "globals.hh" #include "filetransfer.hh" #include "store-api.hh" #include "legacy.hh" #include "fetchers.hh" +#include "util.hh" #include #include @@ -162,11 +164,12 @@ { // Figure out the name of the `.nix-channels' file to use auto home = getHome(); - channelsList = home + "/.nix-channels"; - nixDefExpr = home + "/.nix-defexpr"; + channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + "/channels" : home + "/.nix-channels"; + nixDefExpr = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : home + "/.nix-defexpr"; // Figure out the name of the channels profile. - profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()); + profile = profilesDir() + "/channels"; + createDirs(dirOf(profile)); enum { cNone, diff -Nru nix-2.8.0/src/nix-collect-garbage/nix-collect-garbage.cc nix-2.16.1+dfsg/src/nix-collect-garbage/nix-collect-garbage.cc --- nix-2.8.0/src/nix-collect-garbage/nix-collect-garbage.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix-collect-garbage/nix-collect-garbage.cc 2023-06-06 10:55:23.000000000 +0000 @@ -37,9 +37,10 @@ link = readLink(path); } catch (SysError & e) { if (e.errNo == ENOENT) continue; + throw; } if (link.find("link") != std::string::npos) { - printInfo(format("removing old generations of profile %1%") % path); + printInfo("removing old generations of profile %s", path); if (deleteOlderThan != "") deleteGenerationsOlderThan(path, deleteOlderThan, dryRun); else @@ -76,8 +77,12 @@ return true; }); - auto profilesDir = settings.nixStateDir + "/profiles"; - if (removeOld) removeOldGenerations(profilesDir); + if (removeOld) { + std::set dirsToClean = { + profilesDir(), settings.nixStateDir + "/profiles", dirOf(getDefaultProfile())}; + for (auto & dir : dirsToClean) + removeOldGenerations(dir); + } // Run the actual garbage collector. if (!dryRun) { diff -Nru nix-2.8.0/src/nix-copy-closure/nix-copy-closure.cc nix-2.16.1+dfsg/src/nix-copy-closure/nix-copy-closure.cc --- nix-2.8.0/src/nix-copy-closure/nix-copy-closure.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix-copy-closure/nix-copy-closure.cc 2023-06-06 10:55:23.000000000 +0000 @@ -22,7 +22,7 @@ printVersion("nix-copy-closure"); else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") { if (*arg != "--gzip") - printMsg(lvlError, format("Warning: '%1%' is not implemented, falling back to gzip") % *arg); + warn("'%1%' is not implemented, falling back to gzip", *arg); gzip = true; } else if (*arg == "--from") toMode = false; diff -Nru nix-2.8.0/src/nix-env/nix-env.cc nix-2.16.1+dfsg/src/nix-env/nix-env.cc --- nix-2.8.0/src/nix-env/nix-env.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix-env/nix-env.cc 2023-06-06 10:55:23.000000000 +0000 @@ -12,7 +12,6 @@ #include "local-fs-store.hh" #include "user-env.hh" #include "util.hh" -#include "json.hh" #include "value-to-json.hh" #include "xml-writer.hh" #include "legacy.hh" @@ -26,6 +25,7 @@ #include #include #include +#include using namespace nix; using std::cout; @@ -44,7 +44,7 @@ struct InstallSourceInfo { InstallSourceType type; - Path nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ + std::shared_ptr nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ Path profile; /* for srcProfile */ std::string systemFilter; /* for srcNixExprDrvs */ Bindings * autoArgs; @@ -92,9 +92,11 @@ } -static bool isNixExpr(const Path & path, struct stat & st) +static bool isNixExpr(const SourcePath & path, struct InputAccessor::Stat & st) { - return S_ISREG(st.st_mode) || (S_ISDIR(st.st_mode) && pathExists(path + "/default.nix")); + return + st.type == InputAccessor::tRegular + || (st.type == InputAccessor::tDirectory && (path + "default.nix").pathExists()); } @@ -102,10 +104,10 @@ static void getAllExprs(EvalState & state, - const Path & path, StringSet & seen, BindingsBuilder & attrs) + const SourcePath & path, StringSet & seen, BindingsBuilder & attrs) { StringSet namesSorted; - for (auto & i : readDirectory(path)) namesSorted.insert(i.name); + for (auto & [name, _] : path.readDirectory()) namesSorted.insert(name); for (auto & i : namesSorted) { /* Ignore the manifest.nix used by profiles. This is @@ -113,13 +115,16 @@ are implemented using profiles). */ if (i == "manifest.nix") continue; - Path path2 = path + "/" + i; + SourcePath path2 = path + i; - struct stat st; - if (stat(path2.c_str(), &st) == -1) + InputAccessor::Stat st; + try { + st = path2.resolveSymlinks().lstat(); + } catch (Error &) { continue; // ignore dangling symlinks in ~/.nix-defexpr + } - if (isNixExpr(path2, st) && (!S_ISREG(st.st_mode) || hasSuffix(path2, ".nix"))) { + if (isNixExpr(path2, st) && (st.type != InputAccessor::tRegular || hasSuffix(path2.baseName(), ".nix"))) { /* Strip off the `.nix' filename suffix (if applicable), otherwise the attribute cannot be selected with the `-A' option. Useful if you want to stick a Nix @@ -129,21 +134,20 @@ attrName = std::string(attrName, 0, attrName.size() - 4); if (!seen.insert(attrName).second) { std::string suggestionMessage = ""; - if (path2.find("channels") != std::string::npos && path.find("channels") != std::string::npos) { + if (path2.path.abs().find("channels") != std::string::npos && path.path.abs().find("channels") != std::string::npos) suggestionMessage = fmt("\nsuggestion: remove '%s' from either the root channels or the user channels", attrName); - } printError("warning: name collision in input Nix expressions, skipping '%1%'" "%2%", path2, suggestionMessage); continue; } /* Load the expression on demand. */ auto vArg = state.allocValue(); - vArg->mkString(path2); + vArg->mkString(path2.path.abs()); if (seen.size() == maxAttrs) throw Error("too many Nix expressions in directory '%1%'", path); attrs.alloc(attrName).mkApp(&state.getBuiltin("import"), vArg); } - else if (S_ISDIR(st.st_mode)) + else if (st.type == InputAccessor::tDirectory) /* `path2' is a directory (with no default.nix in it); recurse into it. */ getAllExprs(state, path2, seen, attrs); @@ -152,11 +156,9 @@ -static void loadSourceExpr(EvalState & state, const Path & path, Value & v) +static void loadSourceExpr(EvalState & state, const SourcePath & path, Value & v) { - struct stat st; - if (stat(path.c_str(), &st) == -1) - throw SysError("getting information about '%1%'", path); + auto st = path.resolveSymlinks().lstat(); if (isNixExpr(path, st)) state.evalFile(path, v); @@ -167,7 +169,7 @@ set flat, not nested, to make it easier for a user to have a ~/.nix-defexpr directory that includes some system-wide directory). */ - else if (S_ISDIR(st.st_mode)) { + else if (st.type == InputAccessor::tDirectory) { auto attrs = state.buildBindings(maxAttrs); attrs.alloc("_combineChannels").mkList(0); StringSet seen; @@ -179,7 +181,7 @@ } -static void loadDerivations(EvalState & state, Path nixExprPath, +static void loadDerivations(EvalState & state, const SourcePath & nixExprPath, std::string systemFilter, Bindings & autoArgs, const std::string & pathPrefix, DrvInfos & elems) { @@ -390,7 +392,7 @@ /* Load the derivations from the (default or specified) Nix expression. */ DrvInfos allElems; - loadDerivations(state, instSource.nixExprPath, + loadDerivations(state, *instSource.nixExprPath, instSource.systemFilter, *instSource.autoArgs, "", allElems); elems = filterBySelector(state, allElems, args, newestOnly); @@ -407,10 +409,10 @@ case srcNixExprs: { Value vArg; - loadSourceExpr(state, instSource.nixExprPath, vArg); + loadSourceExpr(state, *instSource.nixExprPath, vArg); for (auto & i : args) { - Expr * eFun = state.parseExprFromString(i, absPath(".")); + Expr * eFun = state.parseExprFromString(i, state.rootPath(CanonPath::fromCwd())); Value vFun, vTmp; state.eval(eFun, vFun); vTmp.mkApp(&vFun, &vArg); @@ -462,7 +464,7 @@ case srcAttrPath: { Value vRoot; - loadSourceExpr(state, instSource.nixExprPath, vRoot); + loadSourceExpr(state, *instSource.nixExprPath, vRoot); for (auto & i : args) { Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot).first); getDerivations(state, v, "", *instSource.autoArgs, elems, true); @@ -478,9 +480,14 @@ std::vector targets; for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) - targets.push_back(DerivedPath::Built{*drvPath}); + targets.push_back(DerivedPath::Built{ + .drvPath = *drvPath, + .outputs = OutputsSpec::All { }, + }); else - targets.push_back(DerivedPath::Opaque{i.queryOutPath()}); + targets.push_back(DerivedPath::Opaque{ + .path = i.queryOutPath(), + }); printMissing(state.store, targets); } @@ -495,7 +502,7 @@ static void installDerivations(Globals & globals, const Strings & args, const Path & profile) { - debug(format("installing derivations")); + debug("installing derivations"); /* Get the set of user environment elements to be installed. */ DrvInfos newElems, newElemsTmp; @@ -574,7 +581,7 @@ static void upgradeDerivations(Globals & globals, const Strings & args, UpgradeType upgradeType) { - debug(format("upgrading derivations")); + debug("upgrading derivations"); /* Upgrade works as follows: we take all currently installed derivations, and for any derivation matching any selector, look @@ -647,7 +654,7 @@ } else newElems.push_back(i); } catch (Error & e) { - e.addTrace(std::nullopt, "while trying to find an upgrade for '%s'", i.queryName()); + e.addTrace(nullptr, "while trying to find an upgrade for '%s'", i.queryName()); throw; } } @@ -751,14 +758,19 @@ auto drvPath = drv.queryDrvPath(); std::vector paths { drvPath - ? (DerivedPath) (DerivedPath::Built { *drvPath }) - : (DerivedPath) (DerivedPath::Opaque { drv.queryOutPath() }), + ? (DerivedPath) (DerivedPath::Built { + .drvPath = *drvPath, + .outputs = OutputsSpec::All { }, + }) + : (DerivedPath) (DerivedPath::Opaque { + .path = drv.queryOutPath(), + }), }; printMissing(globals.state->store, paths); if (globals.dryRun) return; globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); - debug(format("switching to new user environment")); + debug("switching to new user environment"); Path generation = createGeneration( ref(store2), globals.profile, @@ -911,53 +923,58 @@ static void queryJSON(Globals & globals, std::vector & elems, bool printOutPath, bool printMeta) { - JSONObject topObj(cout, true); + using nlohmann::json; + json topObj = json::object(); for (auto & i : elems) { try { if (i.hasFailed()) continue; - JSONObject pkgObj = topObj.object(i.attrPath); auto drvName = DrvName(i.queryName()); - pkgObj.attr("name", drvName.fullName); - pkgObj.attr("pname", drvName.name); - pkgObj.attr("version", drvName.version); - pkgObj.attr("system", i.querySystem()); - pkgObj.attr("outputName", i.queryOutputName()); + json &pkgObj = topObj[i.attrPath]; + pkgObj = { + {"name", drvName.fullName}, + {"pname", drvName.name}, + {"version", drvName.version}, + {"system", i.querySystem()}, + {"outputName", i.queryOutputName()}, + }; { DrvInfo::Outputs outputs = i.queryOutputs(printOutPath); - JSONObject outputObj = pkgObj.object("outputs"); + json &outputObj = pkgObj["outputs"]; + outputObj = json::object(); for (auto & j : outputs) { if (j.second) - outputObj.attr(j.first, globals.state->store->printStorePath(*j.second)); + outputObj[j.first] = globals.state->store->printStorePath(*j.second); else - outputObj.attr(j.first, nullptr); + outputObj[j.first] = nullptr; } } if (printMeta) { - JSONObject metaObj = pkgObj.object("meta"); + json &metaObj = pkgObj["meta"]; + metaObj = json::object(); StringSet metaNames = i.queryMetaNames(); for (auto & j : metaNames) { - auto placeholder = metaObj.placeholder(j); Value * v = i.queryMeta(j); if (!v) { printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); - placeholder.write(nullptr); + metaObj[j] = nullptr; } else { - PathSet context; - printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); + NixStringContext context; + metaObj[j] = printValueAsJSON(*globals.state, true, *v, noPos, context); } } } } catch (AssertionError & e) { printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName()); } catch (Error & e) { - e.addTrace(std::nullopt, "while querying the derivation named '%1%'", i.queryName()); + e.addTrace(nullptr, "while querying the derivation named '%1%'", i.queryName()); throw; } } + std::cout << topObj.dump(2); } @@ -1015,7 +1032,7 @@ installedElems = queryInstalled(*globals.state, globals.profile); if (source == sAvailable || compareVersions) - loadDerivations(*globals.state, globals.instSource.nixExprPath, + loadDerivations(*globals.state, *globals.instSource.nixExprPath, globals.instSource.systemFilter, *globals.instSource.autoArgs, attrPath, availElems); @@ -1078,7 +1095,7 @@ try { if (i.hasFailed()) continue; - //Activity act(*logger, lvlDebug, format("outputting query result '%1%'") % i.attrPath); + //Activity act(*logger, lvlDebug, "outputting query result '%1%'", i.attrPath); if (globals.prebuiltOnly && !validPaths.count(i.queryOutPath()) && @@ -1214,11 +1231,11 @@ xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nInt) { attrs2["type"] = "int"; - attrs2["value"] = (format("%1%") % v->integer).str(); + attrs2["value"] = fmt("%1%", v->integer); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nFloat) { attrs2["type"] = "float"; - attrs2["value"] = (format("%1%") % v->fpoint).str(); + attrs2["value"] = fmt("%1%", v->fpoint); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nBool) { attrs2["type"] = "bool"; @@ -1241,7 +1258,7 @@ Attr & a(*attrs.find(i.name)); if(a.value->type() != nString) continue; XMLAttrs attrs3; - attrs3["type"] = i.name; + attrs3["type"] = globals.state->symbols[i.name]; attrs3["value"] = a.value->string.s; xml.writeEmptyElement("string", attrs3); } @@ -1257,7 +1274,7 @@ } catch (AssertionError & e) { printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName()); } catch (Error & e) { - e.addTrace(std::nullopt, "while querying the derivation named '%1%'", i.queryName()); + e.addTrace(nullptr, "while querying the derivation named '%1%'", i.queryName()); throw; } } @@ -1274,7 +1291,7 @@ throw UsageError("exactly one argument expected"); Path profile = absPath(opArgs.front()); - Path profileLink = getHome() + "/.nix-profile"; + Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile"; switchLink(profileLink, profile); } @@ -1322,11 +1339,11 @@ for (auto & i : gens) { tm t; if (!localtime_r(&i.creationTime, &t)) throw Error("cannot convert time"); - cout << format("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||\n") - % i.number - % (t.tm_year + 1900) % (t.tm_mon + 1) % t.tm_mday - % t.tm_hour % t.tm_min % t.tm_sec - % (i.number == curGen ? "(current)" : ""); + logger->cout("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||", + i.number, + t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, + t.tm_hour, t.tm_min, t.tm_sec, + i.number == curGen ? "(current)" : ""); } } @@ -1372,25 +1389,27 @@ { Strings opFlags, opArgs; Operation op = 0; - RepairFlag repair = NoRepair; + std::string opName; + bool showHelp = false; std::string file; Globals globals; globals.instSource.type = srcUnknown; - globals.instSource.nixExprPath = getHome() + "/.nix-defexpr"; globals.instSource.systemFilter = "*"; - if (!pathExists(globals.instSource.nixExprPath)) { + Path nixExprPath = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : getHome() + "/.nix-defexpr"; + + if (!pathExists(nixExprPath)) { try { - createDirs(globals.instSource.nixExprPath); + createDirs(nixExprPath); replaceSymlink( - fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), - globals.instSource.nixExprPath + "/channels"); + defaultChannelsDir(), + nixExprPath + "/channels"); if (getuid() != 0) replaceSymlink( - fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), - globals.instSource.nixExprPath + "/channels_root"); + rootChannelsDir(), + nixExprPath + "/channels_root"); } catch (Error &) { } } @@ -1408,37 +1427,59 @@ Operation oldOp = op; if (*arg == "--help") - showManPage("nix-env"); + showHelp = true; else if (*arg == "--version") op = opVersion; - else if (*arg == "--install" || *arg == "-i") + else if (*arg == "--install" || *arg == "-i") { op = opInstall; + opName = "-install"; + } else if (*arg == "--force-name") // undocumented flag for nix-install-package globals.forceName = getArg(*arg, arg, end); - else if (*arg == "--uninstall" || *arg == "-e") + else if (*arg == "--uninstall" || *arg == "-e") { op = opUninstall; - else if (*arg == "--upgrade" || *arg == "-u") + opName = "-uninstall"; + } + else if (*arg == "--upgrade" || *arg == "-u") { op = opUpgrade; - else if (*arg == "--set-flag") + opName = "-upgrade"; + } + else if (*arg == "--set-flag") { op = opSetFlag; - else if (*arg == "--set") + opName = arg->substr(1); + } + else if (*arg == "--set") { op = opSet; - else if (*arg == "--query" || *arg == "-q") + opName = arg->substr(1); + } + else if (*arg == "--query" || *arg == "-q") { op = opQuery; + opName = "-query"; + } else if (*arg == "--profile" || *arg == "-p") globals.profile = absPath(getArg(*arg, arg, end)); else if (*arg == "--file" || *arg == "-f") file = getArg(*arg, arg, end); - else if (*arg == "--switch-profile" || *arg == "-S") + else if (*arg == "--switch-profile" || *arg == "-S") { op = opSwitchProfile; - else if (*arg == "--switch-generation" || *arg == "-G") + opName = "-switch-profile"; + } + else if (*arg == "--switch-generation" || *arg == "-G") { op = opSwitchGeneration; - else if (*arg == "--rollback") + opName = "-switch-generation"; + } + else if (*arg == "--rollback") { op = opRollback; - else if (*arg == "--list-generations") + opName = arg->substr(1); + } + else if (*arg == "--list-generations") { op = opListGenerations; - else if (*arg == "--delete-generations") + opName = arg->substr(1); + } + else if (*arg == "--delete-generations") { op = opDeleteGenerations; + opName = arg->substr(1); + } else if (*arg == "--dry-run") { printInfo("(dry run; not doing anything)"); globals.dryRun = true; @@ -1447,8 +1488,6 @@ globals.instSource.systemFilter = getArg(*arg, arg, end); else if (*arg == "--prebuilt-only" || *arg == "-b") globals.prebuiltOnly = true; - else if (*arg == "--repair") - repair = Repair; else if (*arg != "" && arg->at(0) == '-') { opFlags.push_back(*arg); /* FIXME: hacky */ @@ -1467,15 +1506,18 @@ myArgs.parseCmdline(argvToStrings(argc, argv)); + if (showHelp) showManPage("nix-env" + opName); if (!op) throw UsageError("no operation specified"); auto store = openStore(); globals.state = std::shared_ptr(new EvalState(myArgs.searchPath, store)); - globals.state->repair = repair; + globals.state->repair = myArgs.repair; - if (file != "") - globals.instSource.nixExprPath = lookupFileArg(*globals.state, file); + globals.instSource.nixExprPath = std::make_shared( + file != "" + ? lookupFileArg(*globals.state, file) + : globals.state->rootPath(CanonPath(nixExprPath))); globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); @@ -1485,12 +1527,10 @@ if (globals.profile == "") globals.profile = getDefaultProfile(); - op(globals, opFlags, opArgs); + op(globals, std::move(opFlags), std::move(opArgs)); globals.state->printStats(); - logger->stop(); - return 0; } } diff -Nru nix-2.8.0/src/nix-env/user-env.cc nix-2.16.1+dfsg/src/nix-env/user-env.cc --- nix-2.8.0/src/nix-env/user-env.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix-env/user-env.cc 2023-06-06 10:55:23.000000000 +0000 @@ -19,10 +19,10 @@ DrvInfos elems; if (pathExists(userEnv + "/manifest.json")) throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv); - Path manifestFile = userEnv + "/manifest.nix"; + auto manifestFile = userEnv + "/manifest.nix"; if (pathExists(manifestFile)) { Value v; - state.evalFile(manifestFile, v); + state.evalFile(state.rootPath(CanonPath(manifestFile)), v); Bindings & bindings(*state.allocBindings(0)); getDerivations(state, v, "", bindings, elems, false); } @@ -41,7 +41,7 @@ if (auto drvPath = i.queryDrvPath()) drvsToBuild.push_back({*drvPath}); - debug(format("building user environment dependencies")); + debug("building user environment dependencies"); state.store->buildPaths( toDerivedPaths(drvsToBuild), state.repair ? bmRepair : bmNormal); @@ -106,7 +106,7 @@ the store; we need it for future modifications of the environment. */ std::ostringstream str; - manifest.print(str, true); + manifest.print(state.symbols, str, true); auto manifestFile = state.store->addTextToStore("env-manifest.nix", str.str(), references); @@ -114,14 +114,12 @@ Value envBuilder; state.eval(state.parseExprFromString( #include "buildenv.nix.gen.hh" - , "/"), envBuilder); + , state.rootPath(CanonPath::root)), envBuilder); /* Construct a Nix expression that calls the user environment builder with the manifest as argument. */ auto attrs = state.buildBindings(3); - attrs.alloc("manifest").mkString( - state.store->printStorePath(manifestFile), - {state.store->printStorePath(manifestFile)}); + state.mkStorePathString(manifestFile, attrs.alloc("manifest")); attrs.insert(state.symbols.create("derivations"), &manifest); Value args; args.mkAttrs(attrs); @@ -132,11 +130,11 @@ /* Evaluate it. */ debug("evaluating user environment builder"); state.forceValue(topLevel, [&]() { return topLevel.determinePos(noPos); }); - PathSet context; + NixStringContext context; Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); - auto topLevelDrv = state.coerceToStorePath(*aDrvPath.pos, *aDrvPath.value, context); + auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context, ""); Attr & aOutPath(*topLevel.attrs->find(state.sOutPath)); - auto topLevelOut = state.coerceToStorePath(*aOutPath.pos, *aOutPath.value, context); + auto topLevelOut = state.coerceToStorePath(aOutPath.pos, *aOutPath.value, context, ""); /* Realise the resulting store expression. */ debug("building user environment"); @@ -159,7 +157,7 @@ return false; } - debug(format("switching to new user environment")); + debug("switching to new user environment"); Path generation = createGeneration(ref(store2), profile, topLevelOut); switchLink(profile, generation); } diff -Nru nix-2.8.0/src/nix-env/user-env.hh nix-2.16.1+dfsg/src/nix-env/user-env.hh --- nix-2.8.0/src/nix-env/user-env.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix-env/user-env.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "get-drvs.hh" diff -Nru nix-2.8.0/src/nix-instantiate/nix-instantiate.cc nix-2.16.1+dfsg/src/nix-instantiate/nix-instantiate.cc --- nix-2.8.0/src/nix-instantiate/nix-instantiate.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix-instantiate/nix-instantiate.cc 2023-06-06 10:55:23.000000000 +0000 @@ -31,7 +31,8 @@ bool evalOnly, OutputKind output, bool location, Expr * e) { if (parseOnly) { - std::cout << format("%1%\n") % *e; + e->show(state.symbols, std::cout); + std::cout << "\n"; return; } @@ -42,7 +43,7 @@ Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first); state.forceValue(v, [&]() { return v.determinePos(noPos); }); - PathSet context; + NixStringContext context; if (evalOnly) { Value vRes; if (autoArgs.empty()) @@ -51,11 +52,13 @@ state.autoCallFunction(autoArgs, v, vRes); if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); - else if (output == okJSON) + else if (output == okJSON) { printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); - else { + std::cout << std::endl; + } else { if (strict) state.forceValueDeep(vRes); - std::cout << vRes << std::endl; + vRes.print(state.symbols, std::cout); + std::cout << std::endl; } } else { DrvInfos drvs; @@ -99,7 +102,6 @@ bool strict = false; Strings attrPaths; bool wantsReadWrite = false; - RepairFlag repair = NoRepair; struct MyArgs : LegacyArgs, MixEvalArgs { @@ -137,8 +139,6 @@ xmlOutputSourceLocation = false; else if (*arg == "--strict") strict = true; - else if (*arg == "--repair") - repair = Repair; else if (*arg == "--dry-run") settings.readOnlyMode = true; else if (*arg != "" && arg->at(0) == '-') @@ -157,7 +157,7 @@ auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; auto state = std::make_unique(myArgs.searchPath, evalStore, store); - state->repair = repair; + state->repair = myArgs.repair; Bindings & autoArgs = *myArgs.getAutoArgs(*state); @@ -165,9 +165,11 @@ if (findFile) { for (auto & i : files) { - Path p = state->findFile(i); - if (p == "") throw Error("unable to find '%1%'", i); - std::cout << p << std::endl; + auto p = state->findFile(i); + if (auto fn = p.getPhysicalPath()) + std::cout << fn->abs() << std::endl; + else + throw Error("'%s' has no physical path", p); } return 0; } @@ -181,7 +183,7 @@ for (auto & i : files) { Expr * e = fromArgs - ? state->parseExprFromString(i, absPath(".")) + ? state->parseExprFromString(i, state->rootPath(CanonPath::fromCwd())) : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i)))); processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); diff -Nru nix-2.8.0/src/nix-store/dotgraph.hh nix-2.16.1+dfsg/src/nix-store/dotgraph.hh --- nix-2.8.0/src/nix-store/dotgraph.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix-store/dotgraph.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff -Nru nix-2.8.0/src/nix-store/graphml.cc nix-2.16.1+dfsg/src/nix-store/graphml.cc --- nix-2.8.0/src/nix-store/graphml.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix-store/graphml.cc 2023-06-06 10:55:23.000000000 +0000 @@ -57,7 +57,7 @@ << "\n" - << "" + << "" << "" << "" << "\n"; diff -Nru nix-2.8.0/src/nix-store/graphml.hh nix-2.16.1+dfsg/src/nix-store/graphml.hh --- nix-2.8.0/src/nix-store/graphml.hh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix-store/graphml.hh 2023-06-06 10:55:23.000000000 +0000 @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff -Nru nix-2.8.0/src/nix-store/nix-store.cc nix-2.16.1+dfsg/src/nix-store/nix-store.cc --- nix-2.8.0/src/nix-store/nix-store.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/nix-store/nix-store.cc 2023-06-06 10:55:23.000000000 +0000 @@ -72,11 +72,13 @@ Derivation drv = store->derivationFromPath(path.path); rootNr++; + /* FIXME: Encode this empty special case explicitly in the type. */ if (path.outputs.empty()) for (auto & i : drv.outputs) path.outputs.insert(i.first); PathSet outputs; for (auto & j : path.outputs) { + /* Match outputs of a store path with outputs of the derivation that produces it. */ DerivationOutputs::iterator i = drv.outputs.find(j); if (i == drv.outputs.end()) throw Error("derivation '%s' does not have an output named '%s'", @@ -141,6 +143,7 @@ toDerivedPaths(paths), willBuild, willSubstitute, unknown, downloadSize, narSize); + /* Filter out unknown paths from `paths`. */ if (ignoreUnknown) { std::vector paths2; for (auto & i : paths) @@ -201,10 +204,10 @@ /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -215,7 +218,13 @@ std::string hash = *i++; std::string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { + .hash = { + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), + }, + .references = {}, + }))); } @@ -274,17 +283,17 @@ static void opQuery(Strings opFlags, Strings opArgs) { enum QueryType - { qDefault, qOutputs, qRequisites, qReferences, qReferrers + { qOutputs, qRequisites, qReferences, qReferrers , qReferrersClosure, qDeriver, qBinding, qHash, qSize , qTree, qGraph, qGraphML, qResolve, qRoots }; - QueryType query = qDefault; + std::optional query; bool useOutput = false; bool includeOutputs = false; bool forceRealise = false; std::string bindingName; for (auto & i : opFlags) { - QueryType prev = query; + std::optional prev = query; if (i == "--outputs") query = qOutputs; else if (i == "--requisites" || i == "-R") query = qRequisites; else if (i == "--references") query = qReferences; @@ -309,15 +318,15 @@ else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true; else if (i == "--include-outputs") includeOutputs = true; else throw UsageError("unknown flag '%1%'", i); - if (prev != qDefault && prev != query) + if (prev && prev != query) throw UsageError("query type '%1%' conflicts with earlier flag", i); } - if (query == qDefault) query = qOutputs; + if (!query) query = qOutputs; RunPager pager; - switch (query) { + switch (*query) { case qOutputs: { for (auto & i : opArgs) { @@ -457,7 +466,7 @@ /* Print each environment variable in the derivation in a format * that can be sourced by the shell. */ for (auto & i : drv.env) - cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second); + logger->cout("export %1%; %1%=%2%\n", i.first, shellEscape(i.second)); /* Also output the arguments. This doesn't preserve whitespace in arguments. */ @@ -516,7 +525,7 @@ if (!store->isValidPath(info->path) || reregister) { /* !!! races */ if (canonicalise) - canonicalisePathMetaData(store->printStorePath(info->path), -1); + canonicalisePathMetaData(store->printStorePath(info->path), {}); if (!hashGiven) { HashResult hash = hashPath(htSHA256, store->printStorePath(info->path)); info->narHash = hash.first; @@ -808,14 +817,23 @@ if (GET_PROTOCOL_MINOR(clientVersion) >= 2) settings.maxLogSize = readNum(in); if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { - settings.buildRepeat = readInt(in); - settings.enforceDeterminism = readInt(in); + auto nrRepeats = readInt(in); + if (nrRepeats != 0) { + throw Error("client requested repeating builds, but this is not currently implemented"); + } + // Ignore 'enforceDeterminism'. It used to be true by + // default, but also only never had any effect when + // `nrRepeats == 0`. We have already asserted that + // `nrRepeats` in fact is 0, so we can safely ignore this + // without doing something other than what the client + // asked for. + readInt(in); + settings.runDiffHook = true; } if (GET_PROTOCOL_MINOR(clientVersion) >= 7) { settings.keepFailed = (bool) readInt(in); } - settings.printRepeatedBuilds = false; }; while (true) { @@ -831,7 +849,7 @@ case cmdQueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = worker_proto::read(*store, in, Phantom {}); + auto paths = WorkerProto::read(*store, in); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -840,19 +858,19 @@ store->substitutePaths(paths); } - worker_proto::write(*store, out, store->queryValidPaths(paths)); + workerProtoWrite(*store, out, store->queryValidPaths(paths)); break; } case cmdQueryPathInfos: { - auto paths = worker_proto::read(*store, in, Phantom {}); + auto paths = WorkerProto::read(*store, in); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - worker_proto::write(*store, out, info->references); + workerProtoWrite(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -880,7 +898,7 @@ case cmdExportPaths: { readInt(in); // obsolete - store->exportPaths(worker_proto::read(*store, in, Phantom {}), out); + store->exportPaths(WorkerProto::read(*store, in), out); break; } @@ -922,20 +940,22 @@ if (GET_PROTOCOL_MINOR(clientVersion) >= 3) out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(clientVersion >= 6)) { - worker_proto::write(*store, out, status.builtOutputs); + if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { + DrvOutputs builtOutputs; + for (auto & [output, realisation] : status.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + workerProtoWrite(*store, out, builtOutputs); } - break; } case cmdQueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(worker_proto::read(*store, in, Phantom {}), + store->computeFSClosure(WorkerProto::read(*store, in), closure, false, includeOutputs); - worker_proto::write(*store, out, closure); + workerProtoWrite(*store, out, closure); break; } @@ -950,10 +970,10 @@ }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, in, Phantom {}); + info.references = WorkerProto::read(*store, in); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseContentAddressOpt(readString(in)); + info.ca = ContentAddress::parseOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); @@ -1012,64 +1032,109 @@ { Strings opFlags, opArgs; Operation op = 0; + bool readFromStdIn = false; + std::string opName; + bool showHelp = false; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { Operation oldOp = op; if (*arg == "--help") - showManPage("nix-store"); + showHelp = true; else if (*arg == "--version") op = opVersion; - else if (*arg == "--realise" || *arg == "--realize" || *arg == "-r") + else if (*arg == "--realise" || *arg == "--realize" || *arg == "-r") { op = opRealise; - else if (*arg == "--add" || *arg == "-A") + opName = "-realise"; + } + else if (*arg == "--add" || *arg == "-A"){ op = opAdd; - else if (*arg == "--add-fixed") + opName = "-add"; + } + else if (*arg == "--add-fixed") { op = opAddFixed; + opName = arg->substr(1); + } else if (*arg == "--print-fixed-path") op = opPrintFixedPath; - else if (*arg == "--delete") + else if (*arg == "--delete") { op = opDelete; - else if (*arg == "--query" || *arg == "-q") + opName = arg->substr(1); + } + else if (*arg == "--query" || *arg == "-q") { op = opQuery; - else if (*arg == "--print-env") + opName = "-query"; + } + else if (*arg == "--print-env") { op = opPrintEnv; - else if (*arg == "--read-log" || *arg == "-l") + opName = arg->substr(1); + } + else if (*arg == "--read-log" || *arg == "-l") { op = opReadLog; - else if (*arg == "--dump-db") + opName = "-read-log"; + } + else if (*arg == "--dump-db") { op = opDumpDB; - else if (*arg == "--load-db") + opName = arg->substr(1); + } + else if (*arg == "--load-db") { op = opLoadDB; + opName = arg->substr(1); + } else if (*arg == "--register-validity") op = opRegisterValidity; else if (*arg == "--check-validity") op = opCheckValidity; - else if (*arg == "--gc") + else if (*arg == "--gc") { op = opGC; - else if (*arg == "--dump") + opName = arg->substr(1); + } + else if (*arg == "--dump") { op = opDump; - else if (*arg == "--restore") + opName = arg->substr(1); + } + else if (*arg == "--restore") { op = opRestore; - else if (*arg == "--export") + opName = arg->substr(1); + } + else if (*arg == "--export") { op = opExport; - else if (*arg == "--import") + opName = arg->substr(1); + } + else if (*arg == "--import") { op = opImport; + opName = arg->substr(1); + } else if (*arg == "--init") op = opInit; - else if (*arg == "--verify") + else if (*arg == "--verify") { op = opVerify; - else if (*arg == "--verify-path") + opName = arg->substr(1); + } + else if (*arg == "--verify-path") { op = opVerifyPath; - else if (*arg == "--repair-path") + opName = arg->substr(1); + } + else if (*arg == "--repair-path") { op = opRepairPath; - else if (*arg == "--optimise" || *arg == "--optimize") + opName = arg->substr(1); + } + else if (*arg == "--optimise" || *arg == "--optimize") { op = opOptimise; - else if (*arg == "--serve") + opName = "-optimise"; + } + else if (*arg == "--serve") { op = opServe; - else if (*arg == "--generate-binary-cache-key") + opName = arg->substr(1); + } + else if (*arg == "--generate-binary-cache-key") { op = opGenerateBinaryCacheKey; + opName = arg->substr(1); + } else if (*arg == "--add-root") gcRoot = absPath(getArg(*arg, arg, end)); + else if (*arg == "--stdin" && !isatty(STDIN_FILENO)) + readFromStdIn = true; else if (*arg == "--indirect") ; else if (*arg == "--no-output") @@ -1082,20 +1147,26 @@ else opArgs.push_back(*arg); + if (readFromStdIn && op != opImport && op != opRestore && op != opServe) { + std::string word; + while (std::cin >> word) { + opArgs.emplace_back(std::move(word)); + }; + } + if (oldOp && oldOp != op) throw UsageError("only one operation may be specified"); return true; }); + if (showHelp) showManPage("nix-store" + opName); if (!op) throw UsageError("no operation specified"); if (op != opDump && op != opRestore) /* !!! hack */ store = openStore(); - op(opFlags, opArgs); - - logger->stop(); + op(std::move(opFlags), std::move(opArgs)); return 0; } diff -Nru nix-2.8.0/src/resolve-system-dependencies/resolve-system-dependencies.cc nix-2.16.1+dfsg/src/resolve-system-dependencies/resolve-system-dependencies.cc --- nix-2.8.0/src/resolve-system-dependencies/resolve-system-dependencies.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/resolve-system-dependencies/resolve-system-dependencies.cc 2023-06-06 10:55:23.000000000 +0000 @@ -157,13 +157,9 @@ uname(&_uname); - auto cacheParentDir = (format("%1%/dependency-maps") % settings.nixStateDir).str(); + auto cacheParentDir = fmt("%1%/dependency-maps", settings.nixStateDir); - cacheDir = (format("%1%/%2%-%3%-%4%") - % cacheParentDir - % _uname.machine - % _uname.sysname - % _uname.release).str(); + cacheDir = fmt("%1%/%2%-%3%-%4%", cacheParentDir, _uname.machine, _uname.sysname, _uname.release); mkdir(cacheParentDir.c_str(), 0755); mkdir(cacheDir.c_str(), 0755); @@ -176,7 +172,7 @@ impurePaths.insert(argv[2]); else { auto drv = store->derivationFromPath(store->parseStorePath(argv[1])); - impurePaths = tokenizeString(get(drv.env, "__impureHostDeps").value_or("")); + impurePaths = tokenizeString(getOr(drv.env, "__impureHostDeps", "")); impurePaths.insert("/usr/lib/libSystem.dylib"); } diff -Nru nix-2.8.0/src/toml11/LICENSE nix-2.16.1+dfsg/src/toml11/LICENSE --- nix-2.8.0/src/toml11/LICENSE 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/LICENSE 1970-01-01 00:00:00.000000000 +0000 @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2017 Toru Niina - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff -Nru nix-2.8.0/src/toml11/README.md nix-2.16.1+dfsg/src/toml11/README.md --- nix-2.8.0/src/toml11/README.md 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/README.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,1966 +0,0 @@ -toml11 -====== - -[![Build Status on GitHub Actions](https://github.com/ToruNiina/toml11/workflows/build/badge.svg)](https://github.com/ToruNiina/toml11/actions) -[![Build Status on TravisCI](https://travis-ci.org/ToruNiina/toml11.svg?branch=master)](https://travis-ci.org/ToruNiina/toml11) -[![Build status on Appveyor](https://ci.appveyor.com/api/projects/status/m2n08a926asvg5mg/branch/master?svg=true)](https://ci.appveyor.com/project/ToruNiina/toml11/branch/master) -[![Build status on CircleCI](https://circleci.com/gh/ToruNiina/toml11/tree/master.svg?style=svg)](https://circleci.com/gh/ToruNiina/toml11/tree/master) -[![Version](https://img.shields.io/github/release/ToruNiina/toml11.svg?style=flat)](https://github.com/ToruNiina/toml11/releases) -[![License](https://img.shields.io/github/license/ToruNiina/toml11.svg?style=flat)](LICENSE) -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1209136.svg)](https://doi.org/10.5281/zenodo.1209136) - -toml11 is a C++11 (or later) header-only toml parser/encoder depending only on C++ standard library. - -- It is compatible to the latest version of [TOML v1.0.0](https://toml.io/en/v1.0.0). -- It is one of the most TOML standard compliant libraries, tested with [the language agnostic test suite for TOML parsers by BurntSushi](https://github.com/BurntSushi/toml-test). -- It shows highly informative error messages. You can see the error messages about invalid files at [CircleCI](https://circleci.com/gh/ToruNiina/toml11). -- It has configurable container. You can use any random-access containers and key-value maps as backend containers. -- It optionally preserves comments without any overhead. -- It has configurable serializer that supports comments, inline tables, literal strings and multiline strings. -- It supports user-defined type conversion from/into toml values. -- It correctly handles UTF-8 sequences, with or without BOM, both on posix and Windows. - -## Example - -```cpp -#include -#include - -int main() -{ - // ```toml - // title = "an example toml file" - // nums = [3, 1, 4, 1, 5] - // ``` - auto data = toml::parse("example.toml"); - - // find a value with the specified type from a table - std::string title = toml::find(data, "title"); - - // convert the whole array into any container automatically - std::vector nums = toml::find>(data, "nums"); - - // access with STL-like manner - if(!data.contains("foo")) - { - data["foo"] = "bar"; - } - - // pass a fallback - std::string name = toml::find_or(data, "name", "not found"); - - // width-dependent formatting - std::cout << std::setw(80) << data << std::endl; - - return 0; -} -``` - -## Table of Contents - -- [Integration](#integration) -- [Decoding a toml file](#decoding-a-toml-file) - - [In the case of syntax error](#in-the-case-of-syntax-error) - - [Invalid UTF-8 Codepoints](#invalid-utf-8-codepoints) -- [Finding a toml value](#finding-a-toml-value) - - [Finding a value in a table](#finding-a-value-in-a-table) - - [In case of error](#in-case-of-error) - - [Dotted keys](#dotted-keys) -- [Casting a toml value](#casting-a-toml-value) -- [Checking value type](#checking-value-type) -- [More about conversion](#more-about-conversion) - - [Converting an array](#converting-an-array) - - [Converting a table](#converting-a-table) - - [Getting an array of tables](#getting-an-array-of-tables) - - [Cost of conversion](#cost-of-conversion) - - [Converting datetime and its variants](#converting-datetime-and-its-variants) -- [Getting with a fallback](#getting-with-a-fallback) -- [Expecting conversion](#expecting-conversion) -- [Visiting a toml::value](#visiting-a-tomlvalue) -- [Constructing a toml::value](#constructing-a-tomlvalue) -- [Preserving Comments](#preserving-comments) -- [Customizing containers](#customizing-containers) -- [TOML literal](#toml-literal) -- [Conversion between toml value and arbitrary types](#conversion-between-toml-value-and-arbitrary-types) -- [Formatting user-defined error messages](#formatting-user-defined-error-messages) -- [Obtaining location information](#obtaining-location-information) -- [Exceptions](#exceptions) -- [Colorize Error Messages](#colorize-error-messages) -- [Serializing TOML data](#serializing-toml-data) -- [Underlying types](#underlying-types) -- [Unreleased TOML features](#unreleased-toml-features) -- [Breaking Changes from v2](#breaking-changes-from-v2) -- [Running Tests](#running-tests) -- [Contributors](#contributors) -- [Licensing Terms](#licensing-terms) - -## Integration - -Just include the file after adding it to the include path. - -```cpp -#include // that's all! now you can use it. -#include - -int main() -{ - const auto data = toml::parse("example.toml"); - const auto title = toml::find(data, "title"); - std::cout << "the title is " << title << std::endl; - return 0; -} -``` - -The convenient way is to add this repository as a git-submodule or to install -it in your system by CMake. - -Note for MSVC: We recommend to set `/Zc:__cplusplus` to detect C++ version correctly. - -## Decoding a toml file - -To parse a toml file, the only thing you have to do is -to pass a filename to the `toml::parse` function. - -```cpp -const std::string fname("sample.toml"); -const toml::value data = toml::parse(fname); -``` - -As required by the TOML specification, the top-level value is always a table. -You can find a value inside it, cast it into a table explicitly, and insert it as a value into other `toml::value`. - -If it encounters an error while opening a file, it will throw `std::runtime_error`. - -You can also pass a `std::istream` to the `toml::parse` function. -To show a filename in an error message, however, it is recommended to pass the -filename with the stream. - -```cpp -std::ifstream ifs("sample.toml", std::ios_base::binary); -assert(ifs.good()); -const auto data = toml::parse(ifs, /*optional -> */ "sample.toml"); -``` - -**Note**: When you are **on Windows, open a file in binary mode**. -If a file is opened in text-mode, CRLF ("\r\n") will automatically be -converted to LF ("\n") and this causes inconsistency between file size -and the contents that would be read. This causes weird error. - -### In the case of syntax error - -If there is a syntax error in a toml file, `toml::parse` will throw -`toml::syntax_error` that inherits `std::exception`. - -toml11 has clean and informative error messages inspired by Rust and -it looks like the following. - -```console -terminate called after throwing an instance of 'toml::syntax_error' - what(): [error] toml::parse_table: invalid line format # error description - --> example.toml # file name - 3 | a = 42 = true # line num and content - | ^------ expected newline, but got '='. # error reason -``` - -If you (mistakenly) duplicate tables and got an error, it is helpful to see -where they are. toml11 shows both at the same time like the following. - -```console -terminate called after throwing an instance of 'toml::syntax_error' - what(): [error] toml::insert_value: table ("table") already exists. - --> duplicate-table.toml - 1 | [table] - | ~~~~~~~ table already exists here - ... - 3 | [table] - | ~~~~~~~ table defined twice -``` - -When toml11 encounters a malformed value, it tries to detect what type it is. -Then it shows hints to fix the format. An error message while reading one of -the malformed files in [the language agnostic test suite](https://github.com/BurntSushi/toml-test). -is shown below. - -```console -what(): [error] bad time: should be HH:MM:SS.subsec - --> ./datetime-malformed-no-secs.toml - 1 | no-secs = 1987-07-05T17:45Z - | ^------- HH:MM:SS.subsec - | -Hint: pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999 -Hint: fail: 1979-05-27T7:32:00, 1979-05-27 17:32 -``` - -You can find other examples in a job named `output_result` on -[CircleCI](https://circleci.com/gh/ToruNiina/toml11). - -Since the error message generation is generally a difficult task, the current -status is not ideal. If you encounter a weird error message, please let us know -and contribute to improve the quality! - -### Invalid UTF-8 codepoints - -It throws `syntax_error` if a value of an escape sequence -representing unicode character is not a valid UTF-8 codepoint. - -```console - what(): [error] toml::read_utf8_codepoint: input codepoint is too large. - --> utf8.toml - 1 | exceeds_unicode = "\U0011FFFF example" - | ^--------- should be in [0x00..0x10FFFF] -``` - -## Finding a toml value - -After parsing successfully, you can obtain the values from the result of -`toml::parse` using `toml::find` function. - -```toml -# sample.toml -answer = 42 -pi = 3.14 -numbers = [1,2,3] -time = 1979-05-27T07:32:00Z -``` - -``` cpp -const auto data = toml::parse("sample.toml"); -const auto answer = toml::find(data, "answer"); -const auto pi = toml::find(data, "pi"); -const auto numbers = toml::find>(data, "numbers"); -const auto timepoint = toml::find(data, "time"); -``` - -By default, `toml::find` returns a `toml::value`. - -```cpp -const toml::value& answer = toml::find(data, "answer"); -``` - -When you pass an exact TOML type that does not require type conversion, -`toml::find` returns a reference without copying the value. - -```cpp -const auto data = toml::parse("sample.toml"); -const auto& answer = toml::find(data, "answer"); -``` - -If the specified type requires conversion, you can't take a reference to the value. -See also [underlying types](#underlying-types). - -**NOTE**: For some technical reason, automatic conversion between `integer` and -`floating` is not supported. If you want to get a floating value even if a value -has integer value, you need to convert it manually after obtaining a value, -like the following. - -```cpp -const auto vx = toml::find(data, "x"); -double x = vx.is_floating() ? vx.as_floating(std::nothrow) : - static_cast(vx.as_integer()); // it throws if vx is neither - // floating nor integer. -``` - -### Finding a value in a table - -There are several way to get a value defined in a table. -First, you can get a table as a normal value and find a value from the table. - -```toml -[fruit] -name = "apple" -[fruit.physical] -color = "red" -shape = "round" -``` - -``` cpp -const auto data = toml::parse("fruit.toml"); -const auto& fruit = toml::find(data, "fruit"); -const auto name = toml::find(fruit, "name"); - -const auto& physical = toml::find(fruit, "physical"); -const auto color = toml::find(physical, "color"); -const auto shape = toml::find(physical, "shape"); -``` - -Here, variable `fruit` is a `toml::value` and can be used as the first argument -of `toml::find`. - -Second, you can pass as many arguments as the number of subtables to `toml::find`. - -```cpp -const auto data = toml::parse("fruit.toml"); -const auto color = toml::find(data, "fruit", "physical", "color"); -const auto shape = toml::find(data, "fruit", "physical", "shape"); -``` - -### Finding a value in an array - -You can find n-th value in an array by `toml::find`. - -```toml -values = ["foo", "bar", "baz"] -``` - -``` cpp -const auto data = toml::parse("sample.toml"); -const auto values = toml::find(data, "values"); -const auto bar = toml::find(values, 1); -``` - -`toml::find` can also search array recursively. - -```cpp -const auto data = toml::parse("fruit.toml"); -const auto bar = toml::find(data, "values", 1); -``` - -Before calling `toml::find`, you can check if a value corresponding to a key -exists. You can use both `bool toml::value::contains(const key&) const` and -`std::size_t toml::value::count(const key&) const`. Those behaves like the -`std::map::contains` and `std::map::count`. - -```cpp -const auto data = toml::parse("fruit.toml"); -if(data.contains("fruit") && data.at("fruit").count("physical") != 0) -{ - // ... -} -``` - -### In case of error - -If the value does not exist, `toml::find` throws `std::out_of_range` with the -location of the table. - -```console -terminate called after throwing an instance of 'std::out_of_range' - what(): [error] key "answer" not found - --> example.toml - 6 | [tab] - | ~~~~~ in this table -``` - ----- - -If the specified type differs from the actual value contained, it throws -`toml::type_error` that inherits `std::exception`. - -Similar to the case of syntax error, toml11 also displays clean error messages. -The error message when you choose `int` to get `string` value would be like this. - -```console -terminate called after throwing an instance of 'toml::type_error' - what(): [error] toml::value bad_cast to integer - --> example.toml - 3 | title = "TOML Example" - | ~~~~~~~~~~~~~~ the actual type is string -``` - -**NOTE**: In order to show this kind of error message, all the toml values have -a pointer to represent its range in a file. The entire contents of a file is -shared by `toml::value`s and remains on the heap memory. It is recommended to -destruct all the `toml::value` classes after configuring your application -if you have a large TOML file compared to the memory resource. - -### Dotted keys - -TOML v0.5.0 has a new feature named "dotted keys". -You can chain keys to represent the structure of the data. - -```toml -physical.color = "orange" -physical.shape = "round" -``` - -This is equivalent to the following. - -```toml -[physical] -color = "orange" -shape = "round" -``` - -You can get both of the above tables with the same c++ code. - -```cpp -const auto physical = toml::find(data, "physical"); -const auto color = toml::find(physical, "color"); -``` - -The following code does not work for the above toml file. - -```cpp -// XXX this does not work! -const auto color = toml::find(data, "physical.color"); -``` - -The above code works with the following toml file. - -```toml -"physical.color" = "orange" -# equivalent to {"physical.color": "orange"}, -# NOT {"physical": {"color": "orange"}}. -``` - - -## Casting a toml value - -### `toml::get` - -`toml::parse` returns `toml::value`. `toml::value` is a union type that can -contain one of the following types. - -- `toml::boolean` (`bool`) -- `toml::integer` (`std::int64_t`) -- `toml::floating` (`double`) -- `toml::string` (a type convertible to std::string) -- `toml::local_date` -- `toml::local_time` -- `toml::local_datetime` -- `toml::offset_datetime` -- `toml::array` (by default, `std::vector`) - - It depends. See [customizing containers](#customizing-containers) for detail. -- `toml::table` (by default, `std::unordered_map`) - - It depends. See [customizing containers](#customizing-containers) for detail. - -To get a value inside, you can use `toml::get()`. The usage is the same as -`toml::find` (actually, `toml::find` internally uses `toml::get` after casting -a value to `toml::table`). - -``` cpp -const toml::value data = toml::parse("sample.toml"); -const toml::value answer_ = toml::get(data).at("answer"); -const std::int64_t answer = toml::get(answer_); -``` - -When you pass an exact TOML type that does not require type conversion, -`toml::get` returns a reference through which you can modify the content -(if the `toml::value` is `const`, it returns `const` reference). - -```cpp -toml::value data = toml::parse("sample.toml"); -toml::value answer_ = toml::get(data).at("answer"); -toml::integer& answer = toml::get(answer_); -answer = 6 * 9; // write to data.answer. now `answer_` contains 54. -``` - -If the specified type requires conversion, you can't take a reference to the value. -See also [underlying types](#underlying-types). - -It also throws a `toml::type_error` if the type differs. - -### `as_xxx` - -You can also use a member function to cast a value. - -```cpp -const std::int64_t answer = data.as_table().at("answer").as_integer(); -``` - -It also throws a `toml::type_error` if the type differs. If you are sure that -the value `v` contains a value of the specified type, you can suppress checking -by passing `std::nothrow`. - -```cpp -const auto& answer = data.as_table().at("answer"); -if(answer.is_integer() && answer.as_integer(std::nothrow) == 42) -{ - std::cout << "value is 42" << std::endl; -} -``` - -If `std::nothrow` is passed, the functions are marked as noexcept. - -By casting a `toml::value` into an array or a table, you can iterate over the -elements. - -```cpp -const auto data = toml::parse("example.toml"); -std::cout << "keys in the top-level table are the following: \n"; -for(const auto& [k, v] : data.as_table()) -{ - std::cout << k << '\n'; -} - -const auto& fruits = toml::find(data, "fruits"); -for(const auto& v : fruits.as_array()) -{ - std::cout << toml::find(v, "name") << '\n'; -} -``` - -The full list of the functions is below. - -```cpp -namespace toml { -class value { - // ... - const boolean& as_boolean() const&; - const integer& as_integer() const&; - const floating& as_floating() const&; - const string& as_string() const&; - const offset_datetime& as_offset_datetime() const&; - const local_datetime& as_local_datetime() const&; - const local_date& as_local_date() const&; - const local_time& as_local_time() const&; - const array& as_array() const&; - const table& as_table() const&; - // -------------------------------------------------------- - // non-const version - boolean& as_boolean() &; - // ditto... - // -------------------------------------------------------- - // rvalue version - boolean&& as_boolean() &&; - // ditto... - - // -------------------------------------------------------- - // noexcept versions ... - const boolean& as_boolean(const std::nothrow_t&) const& noexcept; - boolean& as_boolean(const std::nothrow_t&) & noexcept; - boolean&& as_boolean(const std::nothrow_t&) && noexcept; - // ditto... -}; -} // toml -``` - -### `at()` - -You can access to the element of a table and an array by `toml::basic_value::at`. - -```cpp -const toml::value v{1,2,3,4,5}; -std::cout << v.at(2).as_integer() << std::endl; // 3 - -const toml::value v{{"foo", 42}, {"bar", 3.14}}; -std::cout << v.at("foo").as_integer() << std::endl; // 42 -``` - -If an invalid key (integer for a table, string for an array), it throws -`toml::type_error` for the conversion. If the provided key is out-of-range, -it throws `std::out_of_range`. - -Note that, although `std::string` has `at()` member function, `toml::value::at` -throws if the contained type is a string. Because `std::string` does not -contain `toml::value`. - -### `operator[]` - -You can also access to the element of a table and an array by -`toml::basic_value::operator[]`. - -```cpp -const toml::value v{1,2,3,4,5}; -std::cout << v[2].as_integer() << std::endl; // 3 - -const toml::value v{{"foo", 42}, {"bar", 3.14}}; -std::cout << v["foo"].as_integer() << std::endl; // 42 -``` - -When you access to a `toml::value` that is not initialized yet via -`operator[](const std::string&)`, the `toml::value` will be a table, -just like the `std::map`. - -```cpp -toml::value v; // not initialized as a table. -v["foo"] = 42; // OK. `v` will be a table. -``` - -Contrary, if you access to a `toml::value` that contains an array via `operator[]`, -it does not check anything. It converts `toml::value` without type check and then -access to the n-th element without boundary check, just like the `std::vector::operator[]`. - -```cpp -toml::value v; // not initialized as an array -v[2] = 42; // error! UB -``` - -Please make sure that the `toml::value` has an array inside when you access to -its element via `operator[]`. - -## Checking value type - -You can check the type of a value by `is_xxx` function. - -```cpp -const toml::value v = /* ... */; -if(v.is_integer()) -{ - std::cout << "value is an integer" << std::endl; -} -``` - -The complete list of the functions is below. - -```cpp -namespace toml { -class value { - // ... - bool is_boolean() const noexcept; - bool is_integer() const noexcept; - bool is_floating() const noexcept; - bool is_string() const noexcept; - bool is_offset_datetime() const noexcept; - bool is_local_datetime() const noexcept; - bool is_local_date() const noexcept; - bool is_local_time() const noexcept; - bool is_array() const noexcept; - bool is_table() const noexcept; - bool is_uninitialized() const noexcept; - // ... -}; -} // toml -``` - -Also, you can get `enum class value_t` from `toml::value::type()`. - -```cpp -switch(data.at("something").type()) -{ - case toml::value_t::integer: /*do some stuff*/ ; break; - case toml::value_t::floating: /*do some stuff*/ ; break; - case toml::value_t::string : /*do some stuff*/ ; break; - default : throw std::runtime_error( - "unexpected type : " + toml::stringize(data.at("something").type())); -} -``` - -The complete list of the `enum`s can be found in the section -[underlying types](#underlying-types). - -The `enum`s can be used as a parameter of `toml::value::is` function like the following. - -```cpp -toml::value v = /* ... */; -if(v.is(toml::value_t::boolean)) // ... -``` - -## More about conversion - -Since `toml::find` internally uses `toml::get`, all the following examples work -with both `toml::get` and `toml::find`. - -### Converting an array - -You can get any kind of `container` class from a `toml::array` -except for `map`-like classes. - -``` cpp -// # sample.toml -// numbers = [1,2,3] - -const auto numbers = toml::find(data, "numbers"); - -const auto vc = toml::get >(numbers); -const auto ls = toml::get >(numbers); -const auto dq = toml::get >(numbers); -const auto ar = toml::get>(numbers); -// if the size of data.at("numbers") is larger than that of std::array, -// it will throw toml::type_error because std::array is not resizable. -``` - -Surprisingly, you can convert `toml::array` into `std::pair` and `std::tuple`. - -```cpp -// numbers = [1,2,3] -const auto tp = toml::get>(numbers); -``` - -This functionality is helpful when you have a toml file like the following. - -```toml -array_of_arrays = [[1, 2, 3], ["foo", "bar", "baz"]] # toml allows this -``` - -What is the corresponding C++ type? -Obviously, it is a `std::pair` of `std::vector`s. - -```cpp -const auto array_of_arrays = toml::find(data, "array_of_arrays"); -const auto aofa = toml::get< - std::pair, std::vector> - >(array_of_arrays); -``` - -If you don't know the type of the elements, you can use `toml::array`, -which is a `std::vector` of `toml::value`, instead. - -```cpp -const auto a_of_a = toml::get(array_of_arrays); -const auto first = toml::get>(a_of_a.at(0)); -``` - -You can change the implementation of `toml::array` with `std::deque` or some -other array-like container. See [Customizing containers](#customizing-containers) -for detail. - -### Converting a table - -When all the values of the table have the same type, toml11 allows you to -convert a `toml::table` to a `map` that contains the convertible type. - -```toml -[tab] -key1 = "foo" # all the values are -key2 = "bar" # toml String -``` - -```cpp -const auto data = toml::parse("sample.toml"); -const auto tab = toml::find>(data, "tab"); -std::cout << tab["key1"] << std::endl; // foo -std::cout << tab["key2"] << std::endl; // bar -``` - -But since `toml::table` is just an alias of `std::unordered_map`, -normally you don't need to convert it because it has all the functionalities that -`std::unordered_map` has (e.g. `operator[]`, `count`, and `find`). In most cases -`toml::table` is sufficient. - -```cpp -toml::table tab = toml::get(data); -if(data.count("title") != 0) -{ - data["title"] = std::string("TOML example"); -} -``` - -You can change the implementation of `toml::table` with `std::map` or some -other map-like container. See [Customizing containers](#customizing-containers) -for detail. - -### Getting an array of tables - -An array of tables is just an array of tables. -You can get it in completely the same way as the other arrays and tables. - -```toml -# sample.toml -array_of_inline_tables = [{key = "value1"}, {key = "value2"}, {key = "value3"}] - -[[array_of_tables]] -key = "value4" -[[array_of_tables]] -key = "value5" -[[array_of_tables]] -key = "value6" -``` - -```cpp -const auto data = toml::parse("sample.toml"); -const auto aot1 = toml::find>(data, "array_of_inline_tables"); -const auto aot2 = toml::find>(data, "array_of_tables"); -``` - -### Cost of conversion - -Although conversion through `toml::(get|find)` is convenient, it has additional -copy-cost because it copies data contained in `toml::value` to the -user-specified type. Of course in some cases this overhead is not ignorable. - -```cpp -// the following code constructs a std::vector. -// it requires heap allocation for vector and element conversion. -const auto array = toml::find>(data, "foo"); -``` - -By passing the exact types, `toml::get` returns reference that has no overhead. - -``` cpp -const auto& tab = toml::find(data, "tab"); -const auto& numbers = toml::find(data, "numbers"); -``` - -Also, `as_xxx` are zero-overhead because they always return a reference. - -``` cpp -const auto& tab = toml::find(data, "tab" ).as_table(); -const auto& numbers = toml::find(data, "numbers").as_array(); -``` - -In this case you need to call `toml::get` each time you access to -the element of `toml::array` because `toml::array` is an array of `toml::value`. - -```cpp -const auto& num0 = toml::get(numbers.at(0)); -const auto& num1 = toml::get(numbers.at(1)); -const auto& num2 = toml::get(numbers.at(2)); -``` - -### Converting datetime and its variants - -TOML v0.5.0 has 4 different datetime objects, `local_date`, `local_time`, -`local_datetime`, and `offset_datetime`. - -Since `local_date`, `local_datetime`, and `offset_datetime` represent a time -point, you can convert them to `std::chrono::system_clock::time_point`. - -Contrary, `local_time` does not represents a time point because they lack a -date information, but it can be converted to `std::chrono::duration` that -represents a duration from the beginning of the day, `00:00:00.000`. - -```toml -# sample.toml -date = 2018-12-23 -time = 12:30:00 -l_dt = 2018-12-23T12:30:00 -o_dt = 2018-12-23T12:30:00+09:30 -``` - -```cpp -const auto data = toml::parse("sample.toml"); - -const auto date = toml::get(data.at("date")); -const auto l_dt = toml::get(data.at("l_dt")); -const auto o_dt = toml::get(data.at("o_dt")); - -const auto time = toml::get(data.at("time")); // 12 * 60 + 30 min -``` - -`local_date` and `local_datetime` are assumed to be in the local timezone when -they are converted into `time_point`. On the other hand, `offset_datetime` only -uses the offset part of the data and it does not take local timezone into account. - -To contain datetime data, toml11 defines its own datetime types. -For more detail, you can see the definitions in [toml/datetime.hpp](toml/datetime.hpp). - -## Getting with a fallback - -`toml::find_or` returns a default value if the value is not found or has a -different type. - -```cpp -const auto data = toml::parse("example.toml"); -const auto num = toml::find_or(data, "num", 42); -``` - -It works recursively if you pass several keys for subtables. -In that case, the last argument is considered to be the optional value. -All other arguments between `toml::value` and the optinoal value are considered as keys. - -```cpp -// [fruit.physical] -// color = "red" -auto data = toml::parse("fruit.toml"); -auto color = toml::find_or(data, "fruit", "physical", "color", "red"); -// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ -// arguments optional value -``` - -Also, `toml::get_or` returns a default value if `toml::get` failed. - -```cpp -toml::value v("foo"); // v contains String -const int value = toml::get_or(v, 42); // conversion fails. it returns 42. -``` - -These functions automatically deduce what type you want to get -from the default value you passed. - -To get a reference through this function, take care about the default value. - -```cpp -toml::value v("foo"); // v contains String -toml::integer& i = toml::get_or(v, 42); // does not work because binding `42` - // to `integer&` is invalid -toml::integer opt = 42; -toml::integer& i = toml::get_or(v, opt); // this works. -``` - -## Expecting conversion - -By using `toml::expect`, you will get your expected value or an error message -without throwing `toml::type_error`. - -```cpp -const auto value = toml::expect(data.at("title")); -if(value.is_ok()) { - std::cout << value.unwrap() << std::endl; -} else { - std::cout << value.unwrap_err() << std::endl; -} -``` - -Also, you can pass a function object to modify the expected value. - -```cpp -const auto value = toml::expect(data.at("number")) - .map(// function that receives expected type (here, int) - [](const int number) -> double { - return number * 1.5 + 1.0; - }).unwrap_or(/*default value =*/ 3.14); -``` - -## Visiting a toml::value - -toml11 provides `toml::visit` to apply a function to `toml::value` in the -same way as `std::variant`. - -```cpp -const toml::value v(3.14); -toml::visit([](const auto& val) -> void { - std::cout << val << std::endl; - }, v); -``` - -The function object that would be passed to `toml::visit` must be able to -receive all the possible TOML types. Also, the result types should be the same -each other. - -## Constructing a toml::value - -`toml::value` can be constructed in various ways. - -```cpp -toml::value v(true); // boolean -toml::value v(42); // integer -toml::value v(3.14); // floating -toml::value v("foobar"); // string -toml::value v(toml::local_date(2019, toml::month_t::Apr, 1)); // date -toml::value v{1, 2, 3, 4, 5}; // array -toml::value v{{"foo", 42}, {"bar", 3.14}, {"baz", "qux"}}; // table -``` - -When constructing a string, you can choose to use either literal or basic string. -By default, it will be a basic string. - -```cpp -toml::value v("foobar", toml::string_t::basic ); -toml::value v("foobar", toml::string_t::literal); -``` - -Datetime objects can be constructed from `std::tm` and -`std::chrono::system_clock::time_point`. But you need to specify what type -you use to avoid ambiguity. - -```cpp -const auto now = std::chrono::system_clock::now(); -toml::value v(toml::local_date(now)); -toml::value v(toml::local_datetime(now)); -toml::value v(toml::offset_datetime(now)); -``` - -Since local time is not equivalent to a time point, because it lacks date -information, it will be constructed from `std::chrono::duration`. - -```cpp -toml::value v(toml::local_time(std::chrono::hours(10))); -``` - -You can construct an array object not only from `initializer_list`, but also -from STL containers. In that case, the element type must be convertible to -`toml::value`. - -```cpp -std::vector vec{1,2,3,4,5}; -toml::value v(vec); -``` - -When you construct an array value, all the elements of `initializer_list` -must be convertible into `toml::value`. - -If a `toml::value` has an array, you can `push_back` an element in it. - -```cpp -toml::value v{1,2,3,4,5}; -v.push_back(6); -``` - -`emplace_back` also works. - -## Preserving comments - -toml11 v3 or later allows you yo choose whether comments are preserved or not via template parameter - -```cpp -const auto data1 = toml::parse("example.toml"); -const auto data2 = toml::parse("example.toml"); -``` - -or macro definition. - -```cpp -#define TOML11_PRESERVE_COMMENTS_BY_DEFAULT -#include -``` - -This feature is controlled by template parameter in `toml::basic_value<...>`. -`toml::value` is an alias of `toml::basic_value<...>`. - -If template parameter is explicitly specified, the return value of `toml::parse` -will be `toml::basic_value`. -If the macro is defined, the alias `toml::value` will be -`toml::basic_value`. - -Comments related to a value can be obtained by `toml::value::comments()`. -The return value has the same interface as `std::vector`. - -```cpp -const auto& com = v.comments(); -for(const auto& c : com) -{ - std::cout << c << std::endl; -} -``` - -Comments just before and just after (within the same line) a value are kept in a value. - -```toml -# this is a comment for v1. -v1 = "foo" - -v2 = "bar" # this is a comment for v2. -# Note that this comment is NOT a comment for v2. - -# this comment is not related to any value -# because there are empty lines between v3. -# this comment will be ignored even if you set `preserve_comments`. - -# this is a comment for v3 -# this is also a comment for v3. -v3 = "baz" # ditto. -``` - -Each comment line becomes one element of a `std::vector`. - -Hash signs will be removed, but spaces after hash sign will not be removed. - -```cpp -v1.comments().at(0) == " this is a comment for v1."s; - -v2.comments().at(1) == " this is a comment for v1."s; - -v3.comments().at(0) == " this is a comment for v3."s; -v3.comments().at(1) == " this is also a comment for v3."s; -v3.comments().at(2) == " ditto."s; -``` - -Note that a comment just after an opening brace of an array will not be a -comment for the array. - -```toml -# this is a comment for a. -a = [ # this is not a comment for a. this will be ignored. - 1, 2, 3, - # this is a comment for `42`. - 42, # this is also a comment for `42`. - 5 -] # this is a comment for a. -``` - -You can also append and modify comments. -The interfaces are the same as `std::vector`. - -```cpp -toml::basic_value v(42); -v.comments().push_back(" add this comment."); -// # add this comment. -// i = 42 -``` - -Also, you can pass a `std::vector` when constructing a -`toml::basic_value`. - -```cpp -std::vector comments{"comment 1", "comment 2"}; -const toml::basic_value v1(42, std::move(comments)); -const toml::basic_value v2(42, {"comment 1", "comment 2"}); -``` - -When `toml::discard_comments` is chosen, comments will not be contained in a value. -`value::comments()` will always be kept empty. -All the modification on comments would be ignored. -All the element access in a `discard_comments` causes the same error as accessing -an element of an empty `std::vector`. - -The comments will also be serialized. If comments exist, those comments will be -added just before the values. - -__NOTE__: Result types from `toml::parse(...)` and -`toml::parse(...)` are different. - -## Customizing containers - -Actually, `toml::basic_value` has 3 template arguments. - -```cpp -template class Table = std::unordered_map, - template class Array = std::vector> -class basic_value; -``` - -This enables you to change the containers used inside. E.g. you can use -`std::map` to contain a table object instead of `std::unordered_map`. -And also can use `std::deque` as a array object instead of `std::vector`. - -You can set these parameters while calling `toml::parse` function. - -```cpp -const auto data = toml::parse< - toml::preserve_comments, std::map, std::deque - >("example.toml"); -``` - -Needless to say, the result types from `toml::parse(...)` and -`toml::parse(...)` are different (unless you specify the same -types as default). - -Note that, since `toml::table` and `toml::array` is an alias for a table and an -array of a default `toml::value`, so it is different from the types actually -contained in a `toml::basic_value` when you customize containers. -To get the actual type in a generic way, use -`typename toml::basic_type::table_type` and -`typename toml::basic_type::array_type`. - -## TOML literal - -toml11 supports `"..."_toml` literal. -It accept both a bare value and a file content. - -```cpp -using namespace toml::literals::toml_literals; - -// `_toml` can convert a bare value without key -const toml::value v = u8"0xDEADBEEF"_toml; -// v is an Integer value containing 0xDEADBEEF. - -// raw string literal (`R"(...)"` is useful for this purpose) -const toml::value t = u8R"( - title = "this is TOML literal" - [table] - key = "value" -)"_toml; -// the literal will be parsed and the result will be contained in t -``` - -The literal function is defined in the same way as the standard library literals -such as `std::literals::string_literals::operator""s`. - -```cpp -namespace toml -{ -inline namespace literals -{ -inline namespace toml_literals -{ -toml::value operator"" _toml(const char* str, std::size_t len); -} // toml_literals -} // literals -} // toml -``` - -Access to the operator can be gained with `using namespace toml::literals;`, -`using namespace toml::toml_literals`, and `using namespace toml::literals::toml_literals`. - -Note that a key that is composed only of digits is allowed in TOML. -And, unlike the file parser, toml-literal allows a bare value without a key. -Thus it is difficult to distinguish arrays having integers and definitions of -tables that are named as digits. -Currently, literal `[1]` becomes a table named "1". -To ensure a literal to be considered as an array with one element, you need to -add a comma after the first element (like `[1,]`). - -```cpp -"[1,2,3]"_toml; // This is an array -"[table]"_toml; // This is a table that has an empty table named "table" inside. -"[[1,2,3]]"_toml; // This is an array of arrays -"[[table]]"_toml; // This is a table that has an array of tables inside. - -"[[1]]"_toml; // This literal is ambiguous. - // Currently, it becomes a table that has array of table "1". -"1 = [{}]"_toml; // This is a table that has an array of table named 1. -"[[1,]]"_toml; // This is an array of arrays. -"[[1],]"_toml; // ditto. -``` - -NOTE: `_toml` literal returns a `toml::value` that does not have comments. - -## Conversion between toml value and arbitrary types - -You can also use `toml::get` and other related functions with the types -you defined after you implement a way to convert it. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; -}; -} // ext - -const auto data = toml::parse("example.toml"); - -// to do this -const foo f = toml::find(data, "foo"); -``` - -There are 3 ways to use `toml::get` with the types that you defined. - -The first one is to implement `from_toml(const toml::value&)` member function. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; - - void from_toml(const toml::value& v) - { - this->a = toml::find(v, "a"); - this->b = toml::find(v, "b"); - this->c = toml::find(v, "c"); - return; - } -}; -} // ext -``` - -In this way, because `toml::get` first constructs `foo` without arguments, -the type should be default-constructible. - -The second is to implement `constructor(const toml::value&)`. - -```cpp -namespace ext -{ -struct foo -{ - explicit foo(const toml::value& v) - : a(toml::find(v, "a")), b(toml::find(v, "b")), - c(toml::find(v, "c")) - {} - - int a; - double b; - std::string c; -}; -} // ext -``` - -Note that implicit default constructor declaration will be suppressed -when a constructor is defined. If you want to use the struct (here, `foo`) -in a container (e.g. `std::vector`), you may need to define default -constructor explicitly. - -The third is to implement specialization of `toml::from` for your type. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; -}; -} // ext - -namespace toml -{ -template<> -struct from -{ - static ext::foo from_toml(const value& v) - { - ext::foo f; - f.a = find(v, "a"); - f.b = find(v, "b"); - f.c = find(v, "c"); - return f; - } -}; -} // toml -``` - -In this way, since the conversion function is defined outside of the class, -you can add conversion between `toml::value` and classes defined in another library. - -In some cases, a class has a templatized constructor that takes a template, `T`. -It confuses `toml::get/find` because it makes the class "constructible" from -`toml::value`. To avoid this problem, `toml::from` and `from_toml` always -precede constructor. It makes easier to implement conversion between -`toml::value` and types defined in other libraries because it skips constructor. - -But, importantly, you cannot define `toml::from` and `T.from_toml` at the same -time because it causes ambiguity in the overload resolution of `toml::get` and `toml::find`. - -So the precedence is `toml::from` == `T.from_toml()` > `T(toml::value)`. - -If you want to convert any versions of `toml::basic_value`, -you need to templatize the conversion function as follows. - -```cpp -struct foo -{ - template class M, template class A> - void from_toml(const toml::basic_value& v) - { - this->a = toml::find(v, "a"); - this->b = toml::find(v, "b"); - this->c = toml::find(v, "c"); - return; - } -}; -// or -namespace toml -{ -template<> -struct from -{ - template class M, template class A> - static ext::foo from_toml(const basic_value& v) - { - ext::foo f; - f.a = find(v, "a"); - f.b = find(v, "b"); - f.c = find(v, "c"); - return f; - } -}; -} // toml -``` - ----- - -The opposite direction is also supported in a similar way. You can directly -pass your type to `toml::value`'s constructor by introducing `into_toml` or -`toml::into`. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; - - toml::value into_toml() const // you need to mark it const. - { - return toml::value{{"a", this->a}, {"b", this->b}, {"c", this->c}}; - } -}; -} // ext - -ext::foo f{42, 3.14, "foobar"}; -toml::value v(f); -``` - -The definition of `toml::into` is similar to `toml::from`. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; -}; -} // ext - -namespace toml -{ -template<> -struct into -{ - static toml::value into_toml(const ext::foo& f) - { - return toml::value{{"a", f.a}, {"b", f.b}, {"c", f.c}}; - } -}; -} // toml - -ext::foo f{42, 3.14, "foobar"}; -toml::value v(f); -``` - -Any type that can be converted to `toml::value`, e.g. `int`, `toml::table` and -`toml::array` are okay to return from `into_toml`. - -You can also return a custom `toml::basic_value` from `toml::into`. - -```cpp -namespace toml -{ -template<> -struct into -{ - static toml::basic_value into_toml(const ext::foo& f) - { - toml::basic_value v{{"a", f.a}, {"b", f.b}, {"c", f.c}}; - v.comments().push_back(" comment"); - return v; - } -}; -} // toml -``` - -But note that, if this `basic_value` would be assigned into other `toml::value` -that discards `comments`, the comments would be dropped. - -### Macro to automatically define conversion functions - -There is a helper macro that automatically generates conversion functions `from` and `into` for a simple struct. - -```cpp -namespace foo -{ -struct Foo -{ - std::string s; - double d; - int i; -}; -} // foo - -TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(foo::Foo, s, d, i) - -int main() -{ - const auto file = toml::parse("example.toml"); - auto f = toml::find(file, "foo"); -} -``` - -And then you can use `toml::find(file, "foo");` - -**Note** that, because of a slight difference in implementation of preprocessor between gcc/clang and MSVC, [you need to define `/Zc:preprocessor`](https://github.com/ToruNiina/toml11/issues/139#issuecomment-803683682) to use it in MSVC (Thank you @glebm !). - -## Formatting user-defined error messages - -When you encounter an error after you read the toml value, you may want to -show the error with the value. - -toml11 provides you a function that formats user-defined error message with -related values. With a code like the following, - -```cpp -const auto value = toml::find(data, "num"); -if(value < 0) -{ - std::cerr << toml::format_error("[error] value should be positive", - data.at("num"), "positive number required") - << std::endl; -} -``` - -you will get an error message like this. - -```console -[error] value should be positive - --> example.toml - 3 | num = -42 - | ~~~ positive number required -``` - -When you pass two values to `toml::format_error`, - -```cpp -const auto min = toml::find(range, "min"); -const auto max = toml::find(range, "max"); -if(max < min) -{ - std::cerr << toml::format_error("[error] max should be larger than min", - data.at("min"), "minimum number here", - data.at("max"), "maximum number here"); - << std::endl; -} -``` - -you will get an error message like this. - -```console -[error] max should be larger than min - --> example.toml - 3 | min = 54 - | ~~ minimum number here - ... - 4 | max = 42 - | ~~ maximum number here -``` - -You can print hints at the end of the message. - -```cpp -std::vector hints; -hints.push_back("positive number means n >= 0."); -hints.push_back("negative number is not positive."); -std::cerr << toml::format_error("[error] value should be positive", - data.at("num"), "positive number required", hints) - << std::endl; -``` - -```console -[error] value should be positive - --> example.toml - 2 | num = 42 - | ~~ positive number required - | -Hint: positive number means n >= 0. -Hint: negative number is not positive. -``` - -## Obtaining location information - -You can also format error messages in your own way by using `source_location`. - -```cpp -struct source_location -{ - std::uint_least32_t line() const noexcept; - std::uint_least32_t column() const noexcept; - std::uint_least32_t region() const noexcept; - std::string const& file_name() const noexcept; - std::string const& line_str() const noexcept; -}; -// +-- line() +--- length of the region (here, region() == 9) -// v .---+---. -// 12 | value = "foo bar" <- line_str() returns the line itself. -// ^-------- column() points here -``` - -You can get this by -```cpp -const toml::value v = /*...*/; -const toml::source_location loc = v.location(); -``` - -## Exceptions - -The following `exception` classes inherits `toml::exception` that inherits -`std::exception`. - -```cpp -namespace toml { -struct exception : public std::exception {/**/}; -struct syntax_error : public toml::exception {/**/}; -struct type_error : public toml::exception {/**/}; -struct internal_error : public toml::exception {/**/}; -} // toml -``` - -`toml::exception` has `toml::exception::location()` member function that returns -`toml::source_location`, in addition to `what()`. - -```cpp -namespace toml { -struct exception : public std::exception -{ - // ... - source_location const& location() const noexcept; -}; -} // toml -``` - -It represents where the error occurs. - -`syntax_error` will be thrown from `toml::parse` and `_toml` literal. -`type_error` will be thrown from `toml::get/find`, `toml::value::as_xxx()`, and -other functions that takes a content inside of `toml::value`. - -Note that, currently, from `toml::value::at()` and `toml::find(value, key)` -may throw an `std::out_of_range` that does not inherits `toml::exception`. - -Also, in some cases, most likely in the file open error, it will throw an -`std::runtime_error`. - -## Colorize Error Messages - -By defining `TOML11_COLORIZE_ERROR_MESSAGE`, the error messages from -`toml::parse` and `toml::find|get` will be colorized. By default, this feature -is turned off. - -With the following toml file taken from `toml-lang/toml/tests/hard_example.toml`, - -```toml -[error] -array = [ - "This might most likely happen in multiline arrays", - Like here, - "or here, - and here" - ] End of array comment, forgot the # -``` - -the error message would be like this. - -![error-message-1](https://github.com/ToruNiina/toml11/blob/misc/misc/toml11-err-msg-1.png) - -With the following, - -```toml -[error] -# array = [ -# "This might most likely happen in multiline arrays", -# Like here, -# "or here, -# and here" -# ] End of array comment, forgot the # -number = 3.14 pi <--again forgot the # -``` - -the error message would be like this. - -![error-message-2](https://github.com/ToruNiina/toml11/blob/misc/misc/toml11-err-msg-2.png) - -The message would be messy when it is written to a file, not a terminal because -it uses [ANSI escape code](https://en.wikipedia.org/wiki/ANSI_escape_code). - -Without `TOML11_COLORIZE_ERROR_MESSAGE`, you can still colorize user-defined -error message by passing `true` to the `toml::format_error` function. -If you define `TOML11_COLORIZE_ERROR_MESSAGE`, the value is `true` by default. -If not, the default value would be `false`. - -```cpp -std::cerr << toml::format_error("[error] value should be positive", - data.at("num"), "positive number required", - hints, /*colorize = */ true) << std::endl; -``` - -Note: It colorize `[error]` in red. That means that it detects `[error]` prefix -at the front of the error message. If there is no `[error]` prefix, -`format_error` adds it to the error message. - -## Serializing TOML data - -toml11 enables you to serialize data into toml format. - -```cpp -const toml::value data{{"foo", 42}, {"bar", "baz"}}; -std::cout << data << std::endl; -// bar = "baz" -// foo = 42 -``` - -toml11 automatically makes a small table and small array inline. -You can specify the width to make them inline by `std::setw` for streams. - -```cpp -const toml::value data{ - {"qux", {{"foo", 42}, {"bar", "baz"}}}, - {"quux", {"small", "array", "of", "strings"}}, - {"foobar", {"this", "array", "of", "strings", "is", "too", "long", - "to", "print", "into", "single", "line", "isn't", "it?"}}, -}; - -// the threshold becomes 80. -std::cout << std::setw(80) << data << std::endl; -// foobar = [ -// "this","array","of","strings","is","too","long","to","print","into", -// "single","line","isn't","it?", -// ] -// quux = ["small","array","of","strings"] -// qux = {bar="baz",foo=42} - - -// the width is 0. nothing become inline. -std::cout << std::setw(0) << data << std::endl; -// foobar = [ -// "this", -// ... (snip) -// "it?", -// ] -// quux = [ -// "small", -// "array", -// "of", -// "strings", -// ] -// [qux] -// bar = "baz" -// foo = 42 -``` - -It is recommended to set width before printing data. Some I/O functions changes -width to 0, and it makes all the stuff (including `toml::array`) multiline. -The resulting files becomes too long. - -To control the precision of floating point numbers, you need to pass -`std::setprecision` to stream. - -```cpp -const toml::value data{ - {"pi", 3.141592653589793}, - {"e", 2.718281828459045} -}; -std::cout << std::setprecision(17) << data << std::endl; -// e = 2.7182818284590451 -// pi = 3.1415926535897931 -std::cout << std::setprecision( 7) << data << std::endl; -// e = 2.718282 -// pi = 3.141593 -``` - -There is another way to format toml values, `toml::format()`. -It returns `std::string` that represents a value. - -```cpp -const toml::value v{{"a", 42}}; -const std::string fmt = toml::format(v); -// a = 42 -``` - -Note that since `toml::format` formats a value, the resulting string may lack -the key value. - -```cpp -const toml::value v{3.14}; -const std::string fmt = toml::format(v); -// 3.14 -``` - -To control the width and precision, `toml::format` receives optional second and -third arguments to set them. By default, the width is 80 and the precision is -`std::numeric_limits::max_digit10`. - -```cpp -const auto serial = toml::format(data, /*width = */ 0, /*prec = */ 17); -``` - -When you pass a comment-preserving-value, the comment will also be serialized. -An array or a table containing a value that has a comment would not be inlined. - -## Underlying types - -The toml types (can be used as `toml::*` in this library) and corresponding `enum` names are listed in the table below. - -| TOML type | underlying c++ type | enum class | -| -------------- | ---------------------------------- | -------------------------------- | -| Boolean | `bool` | `toml::value_t::boolean` | -| Integer | `std::int64_t` | `toml::value_t::integer` | -| Float | `double` | `toml::value_t::floating` | -| String | `toml::string` | `toml::value_t::string` | -| LocalDate | `toml::local_date` | `toml::value_t::local_date` | -| LocalTime | `toml::local_time` | `toml::value_t::local_time` | -| LocalDatetime | `toml::local_datetime` | `toml::value_t::local_datetime` | -| OffsetDatetime | `toml::offset_datetime` | `toml::value_t::offset_datetime` | -| Array | `array-like` | `toml::value_t::array` | -| Table | `map-like` | `toml::value_t::table` | - -`array-like` and `map-like` are the STL containers that works like a `std::vector` and -`std::unordered_map`, respectively. By default, `std::vector` and `std::unordered_map` -are used. See [Customizing containers](#customizing-containers) for detail. - -`toml::string` is effectively the same as `std::string` but has an additional -flag that represents a kind of a string, `string_t::basic` and `string_t::literal`. -Although `std::string` is not an exact toml type, still you can get a reference -that points to internal `std::string` by using `toml::get()` for convenience. -The most important difference between `std::string` and `toml::string` is that -`toml::string` will be formatted as a TOML string when outputted with `ostream`. -This feature is introduced to make it easy to write a custom serializer. - -`Datetime` variants are `struct` that are defined in this library. -Because `std::chrono::system_clock::time_point` is a __time point__, -not capable of representing a Local Time independent from a specific day. - -## Unreleased TOML features - -Since TOML v1.0.0-rc.1 has been released, those features are now activated by -default. We no longer need to define `TOML11_USE_UNRELEASED_FEATURES`. - -- Leading zeroes in exponent parts of floats are permitted. - - e.g. `1.0e+01`, `5e+05` - - [toml-lang/toml/PR/656](https://github.com/toml-lang/toml/pull/656) -- Allow raw tab characters in basic strings and multi-line basic strings. - - [toml-lang/toml/PR/627](https://github.com/toml-lang/toml/pull/627) -- Allow heterogeneous arrays - - [toml-lang/toml/PR/676](https://github.com/toml-lang/toml/pull/676) - -## Note about heterogeneous arrays - -Although `toml::parse` allows heterogeneous arrays, constructor of `toml::value` -does not. Here the reason is explained. - -```cpp -// this won't be compiled -toml::value v{ - "foo", 3.14, 42, {1,2,3,4,5}, {{"key", "value"}} -} -``` - -There is a workaround for this. By explicitly converting values into -`toml::value`, you can initialize `toml::value` with a heterogeneous array. -Also, you can first initialize a `toml::value` with an array and then -`push_back` into it. - -```cpp -// OK! -toml::value v{ - toml::value("foo"), toml::value(3.14), toml::value(42), - toml::value{1,2,3,4,5}, toml::value{{"key", "value"}} -} - -// OK! -toml::value v(toml::array{}); -v.push_back("foo"); -v.push_back(3.14); - -// OK! -toml::array a; -a.push_back("foo"); -a.push_back(3.14); -toml::value v(std::move(a)); -``` - -The reason why the first example is not allowed is the following. -Let's assume that you are initializing a `toml::value` with a table. - -```cpp - // # expecting TOML table. -toml::value v{ // [v] - {"answer", 42}, // answer = 42 - {"pi", 3.14}, // pi = 3.14 - {"foo", "bar"} // foo = "bar" -}; -``` - -This is indistinguishable from a (heterogeneous) TOML array definition. - -```toml -v = [ - ["answer", 42], - ["pi", 3.14], - ["foo", "bar"], -] -``` - -This means that the above C++ code makes constructor's overload resolution -ambiguous. So a constructor that allows both "table as an initializer-list" and -"heterogeneous array as an initializer-list" cannot be implemented. - -Thus, although it is painful, we need to explicitly cast values into -`toml::value` when you initialize heterogeneous array in a C++ code. - -```cpp -toml::value v{ - toml::value("foo"), toml::value(3.14), toml::value(42), - toml::value{1,2,3,4,5}, toml::value{{"key", "value"}} -}; -``` - -## Breaking Changes from v2 - -Although toml11 is relatively new library (it's three years old now), it had -some confusing and inconvenient user-interfaces because of historical reasons. - -Between v2 and v3, those interfaces are rearranged. - -- `toml::parse` now returns a `toml::value`, not `toml::table`. -- `toml::value` is now an alias of `toml::basic_value`. - - See [Customizing containers](#customizing-containers) for detail. -- The elements of `toml::value_t` are renamed as `snake_case`. - - See [Underlying types](#underlying-types) for detail. -- Supports for the CamelCaseNames are dropped. - - See [Underlying types](#underlying-types) for detail. -- `(is|as)_float` has been removed to make the function names consistent with others. - - Since `float` is a keyword, toml11 named a float type as `toml::floating`. - - Also a `value_t` corresponds to `toml::floating` is named `value_t::floating`. - - So `(is|as)_floating` is introduced and `is_float` has been removed. - - See [Casting a toml::value](#casting-a-tomlvalue) and [Checking value type](#checking-value-type) for detail. -- An overload of `toml::find` for `toml::table` has been dropped. Use `toml::value` version instead. - - Because type conversion between a table and a value causes ambiguity while overload resolution - - Since `toml::parse` now returns a `toml::value`, this feature becomes less important. - - Also because `toml::table` is a normal STL container, implementing utility function is easy. - - See [Finding a toml::value](#finding-a-toml-value) for detail. -- An overload of `operator<<` and `toml::format` for `toml::table`s are dropped. - - Use `toml::value` instead. - - See [Serializing TOML data](#serializing-toml-data) for detail. -- Interface around comments. - - See [Preserving Comments](#preserving-comments) for detail. -- An ancient `from_toml/into_toml` has been removed. Use arbitrary type conversion support. - - See [Conversion between toml value and arbitrary types](#conversion-between-toml-value-and-arbitrary-types) for detail. - -Such a big change will not happen in the coming years. - -## Running Tests - -After cloning this repository, run the following command (thank you @jwillikers -for automating test set fetching!). - -```sh -$ mkdir build -$ cd build -$ cmake .. -Dtoml11_BUILD_TEST=ON -$ make -$ make test -``` - -To run the language agnostic test suite, you need to compile -`tests/check_toml_test.cpp` and pass it to the tester. - -## Contributors - -I appreciate the help of the contributors who introduced the great feature to this library. - -- Guillaume Fraux (@Luthaf) - - Windows support and CI on Appvayor - - Intel Compiler support -- Quentin Khan (@xaxousis) - - Found & Fixed a bug around ODR - - Improved error messages for invalid keys to show the location where the parser fails -- Petr Beneš (@wbenny) - - Fixed warnings on MSVC -- Ivan Shynkarenka (@chronoxor) - - Fixed Visual Studio 2019 warnings -- @khoitd1997 - - Fixed warnings while type conversion -- @KerstinKeller - - Added installation script to CMake -- J.C. Moyer (@jcmoyer) - - Fixed an example code in the documentation -- Jt Freeman (@blockparty-sh) - - Fixed feature test macro around `localtime_s` - - Suppress warnings in Debug mode -- OGAWA Kenichi (@kenichiice) - - Suppress warnings on intel compiler -- Jordan Williams (@jwillikers) - - Fixed clang range-loop-analysis warnings - - Fixed feature test macro to suppress -Wundef - - Use cache variables in CMakeLists.txt - - Automate test set fetching, update and refactor CMakeLists.txt -- Scott McCaskill - - Parse 9 digits (nanoseconds) of fractional seconds in a `local_time` -- Shu Wang (@halfelf) - - fix "Finding a value in an array" example in README -- @maass-tv and @SeverinLeonhardt - - Fix MSVC warning C4866 -- OGAWA KenIchi (@kenichiice) - - Fix include path in README -- Mohammed Alyousef (@MoAlyousef) - - Made testing optional in CMake -- Ivan Shynkarenka (@chronoxor) - - Fix compilation error in `` with MinGW -- Alex Merry (@amerry) - - Add missing include files -- sneakypete81 (@sneakypete81) - - Fix typo in error message -- Oliver Kahrmann (@founderio) - - Fix missing filename in error message if parsed file is empty -- Karl Nilsson (@karl-nilsson) - - Fix many spelling errors -- ohdarling88 (@ohdarling) - - Fix a bug in a constructor of serializer -- estshorter (@estshorter) - - Fix MSVC warning C26478 -- Philip Top (@phlptp) - - Improve checking standard library feature availability check -- Louis Marascio (@marascio) - - Fix free-nonheap-object warning - - -## Licensing terms - -This product is licensed under the terms of the [MIT License](LICENSE). - -- Copyright (c) 2017-2021 Toru Niina - -All rights reserved. diff -Nru nix-2.8.0/src/toml11/toml/color.hpp nix-2.16.1+dfsg/src/toml11/toml/color.hpp --- nix-2.8.0/src/toml11/toml/color.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/color.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,64 +0,0 @@ -#ifndef TOML11_COLOR_HPP -#define TOML11_COLOR_HPP -#include -#include - -#ifdef TOML11_COLORIZE_ERROR_MESSAGE -#define TOML11_ERROR_MESSAGE_COLORIZED true -#else -#define TOML11_ERROR_MESSAGE_COLORIZED false -#endif - -namespace toml -{ - -// put ANSI escape sequence to ostream -namespace color_ansi -{ -namespace detail -{ -inline int colorize_index() -{ - static const int index = std::ios_base::xalloc(); - return index; -} -} // detail - -inline std::ostream& colorize(std::ostream& os) -{ - // by default, it is zero. - os.iword(detail::colorize_index()) = 1; - return os; -} -inline std::ostream& nocolorize(std::ostream& os) -{ - os.iword(detail::colorize_index()) = 0; - return os; -} -inline std::ostream& reset (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[00m";} return os;} -inline std::ostream& bold (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[01m";} return os;} -inline std::ostream& grey (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[30m";} return os;} -inline std::ostream& red (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[31m";} return os;} -inline std::ostream& green (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[32m";} return os;} -inline std::ostream& yellow (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[33m";} return os;} -inline std::ostream& blue (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[34m";} return os;} -inline std::ostream& magenta(std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[35m";} return os;} -inline std::ostream& cyan (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[36m";} return os;} -inline std::ostream& white (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[37m";} return os;} -} // color_ansi - -// ANSI escape sequence is the only and default colorization method currently -namespace color = color_ansi; - -} // toml -#endif// TOML11_COLOR_HPP diff -Nru nix-2.8.0/src/toml11/toml/combinator.hpp nix-2.16.1+dfsg/src/toml11/toml/combinator.hpp --- nix-2.8.0/src/toml11/toml/combinator.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/combinator.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,306 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_COMBINATOR_HPP -#define TOML11_COMBINATOR_HPP -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "region.hpp" -#include "result.hpp" -#include "traits.hpp" -#include "utility.hpp" - -// they scans characters and returns region if it matches to the condition. -// when they fail, it does not change the location. -// in lexer.hpp, these are used. - -namespace toml -{ -namespace detail -{ - -// to output character as an error message. -inline std::string show_char(const char c) -{ - // It suppresses an error that occurs only in Debug mode of MSVC++ on Windows. - // I'm not completely sure but they check the value of char to be in the - // range [0, 256) and some of the COMPLETELY VALID utf-8 character sometimes - // has negative value (if char has sign). So here it re-interprets c as - // unsigned char through pointer. In general, converting pointer to a - // pointer that has different type cause UB, but `(signed|unsigned)?char` - // are one of the exceptions. Converting pointer only to char and std::byte - // (c++17) are valid. - if(std::isgraph(*reinterpret_cast(std::addressof(c)))) - { - return std::string(1, c); - } - else - { - std::array buf; - buf.fill('\0'); - const auto r = std::snprintf( - buf.data(), buf.size(), "0x%02x", static_cast(c) & 0xFF); - (void) r; // Unused variable warning - assert(r == static_cast(buf.size()) - 1); - return std::string(buf.data()); - } -} - -template -struct character -{ - static constexpr char target = C; - - static result - invoke(location& loc) - { - if(loc.iter() == loc.end()) {return none();} - const auto first = loc.iter(); - - const char c = *(loc.iter()); - if(c != target) - { - return none(); - } - loc.advance(); // update location - - return ok(region(loc, first, loc.iter())); - } -}; -template -constexpr char character::target; - -// closed interval [Low, Up]. both Low and Up are included. -template -struct in_range -{ - // assuming ascii part of UTF-8... - static_assert(Low <= Up, "lower bound should be less than upper bound."); - - static constexpr char upper = Up; - static constexpr char lower = Low; - - static result - invoke(location& loc) - { - if(loc.iter() == loc.end()) {return none();} - const auto first = loc.iter(); - - const char c = *(loc.iter()); - if(c < lower || upper < c) - { - return none(); - } - - loc.advance(); - return ok(region(loc, first, loc.iter())); - } -}; -template constexpr char in_range::upper; -template constexpr char in_range::lower; - -// keep iterator if `Combinator` matches. otherwise, increment `iter` by 1 char. -// for detecting invalid characters, like control sequences in toml string. -template -struct exclude -{ - static result - invoke(location& loc) - { - if(loc.iter() == loc.end()) {return none();} - auto first = loc.iter(); - - auto rslt = Combinator::invoke(loc); - if(rslt.is_ok()) - { - loc.reset(first); - return none(); - } - loc.reset(std::next(first)); // XXX maybe loc.advance() is okay but... - return ok(region(loc, first, loc.iter())); - } -}; - -// increment `iter`, if matches. otherwise, just return empty string. -template -struct maybe -{ - static result - invoke(location& loc) - { - const auto rslt = Combinator::invoke(loc); - if(rslt.is_ok()) - { - return rslt; - } - return ok(region(loc)); - } -}; - -template -struct sequence; - -template -struct sequence -{ - static result - invoke(location& loc) - { - const auto first = loc.iter(); - auto rslt = Head::invoke(loc); - if(rslt.is_err()) - { - loc.reset(first); - return none(); - } - return sequence::invoke(loc, std::move(rslt.unwrap()), first); - } - - // called from the above function only, recursively. - template - static result - invoke(location& loc, region reg, Iterator first) - { - const auto rslt = Head::invoke(loc); - if(rslt.is_err()) - { - loc.reset(first); - return none(); - } - reg += rslt.unwrap(); // concat regions - return sequence::invoke(loc, std::move(reg), first); - } -}; - -template -struct sequence -{ - // would be called from sequence::invoke only. - template - static result - invoke(location& loc, region reg, Iterator first) - { - const auto rslt = Head::invoke(loc); - if(rslt.is_err()) - { - loc.reset(first); - return none(); - } - reg += rslt.unwrap(); // concat regions - return ok(reg); - } -}; - -template -struct either; - -template -struct either -{ - static result - invoke(location& loc) - { - const auto rslt = Head::invoke(loc); - if(rslt.is_ok()) {return rslt;} - return either::invoke(loc); - } -}; -template -struct either -{ - static result - invoke(location& loc) - { - return Head::invoke(loc); - } -}; - -template -struct repeat; - -template struct exactly{}; -template struct at_least{}; -struct unlimited{}; - -template -struct repeat> -{ - static result - invoke(location& loc) - { - region retval(loc); - const auto first = loc.iter(); - for(std::size_t i=0; i -struct repeat> -{ - static result - invoke(location& loc) - { - region retval(loc); - - const auto first = loc.iter(); - for(std::size_t i=0; i -struct repeat -{ - static result - invoke(location& loc) - { - region retval(loc); - while(true) - { - auto rslt = T::invoke(loc); - if(rslt.is_err()) - { - return ok(std::move(retval)); - } - retval += rslt.unwrap(); - } - } -}; - -} // detail -} // toml -#endif// TOML11_COMBINATOR_HPP diff -Nru nix-2.8.0/src/toml11/toml/comments.hpp nix-2.16.1+dfsg/src/toml11/toml/comments.hpp --- nix-2.8.0/src/toml11/toml/comments.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/comments.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,472 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_COMMENTS_HPP -#define TOML11_COMMENTS_HPP -#include -#include -#include -#include -#include -#include -#include - -#ifdef TOML11_PRESERVE_COMMENTS_BY_DEFAULT -# define TOML11_DEFAULT_COMMENT_STRATEGY ::toml::preserve_comments -#else -# define TOML11_DEFAULT_COMMENT_STRATEGY ::toml::discard_comments -#endif - -// This file provides mainly two classes, `preserve_comments` and `discard_comments`. -// Those two are a container that have the same interface as `std::vector` -// but bahaves in the opposite way. `preserve_comments` is just the same as -// `std::vector` and each `std::string` corresponds to a comment line. -// Conversely, `discard_comments` discards all the strings and ignores everything -// assigned in it. `discard_comments` is always empty and you will encounter an -// error whenever you access to the element. -namespace toml -{ -struct discard_comments; // forward decl - -// use it in the following way -// -// const toml::basic_value data = -// toml::parse("example.toml"); -// -// the interface is almost the same as std::vector. -struct preserve_comments -{ - // `container_type` is not provided in discard_comments. - // do not use this inner-type in a generic code. - using container_type = std::vector; - - using size_type = container_type::size_type; - using difference_type = container_type::difference_type; - using value_type = container_type::value_type; - using reference = container_type::reference; - using const_reference = container_type::const_reference; - using pointer = container_type::pointer; - using const_pointer = container_type::const_pointer; - using iterator = container_type::iterator; - using const_iterator = container_type::const_iterator; - using reverse_iterator = container_type::reverse_iterator; - using const_reverse_iterator = container_type::const_reverse_iterator; - - preserve_comments() = default; - ~preserve_comments() = default; - preserve_comments(preserve_comments const&) = default; - preserve_comments(preserve_comments &&) = default; - preserve_comments& operator=(preserve_comments const&) = default; - preserve_comments& operator=(preserve_comments &&) = default; - - explicit preserve_comments(const std::vector& c): comments(c){} - explicit preserve_comments(std::vector&& c) - : comments(std::move(c)) - {} - preserve_comments& operator=(const std::vector& c) - { - comments = c; - return *this; - } - preserve_comments& operator=(std::vector&& c) - { - comments = std::move(c); - return *this; - } - - explicit preserve_comments(const discard_comments&) {} - - explicit preserve_comments(size_type n): comments(n) {} - preserve_comments(size_type n, const std::string& x): comments(n, x) {} - preserve_comments(std::initializer_list x): comments(x) {} - template - preserve_comments(InputIterator first, InputIterator last) - : comments(first, last) - {} - - template - void assign(InputIterator first, InputIterator last) {comments.assign(first, last);} - void assign(std::initializer_list ini) {comments.assign(ini);} - void assign(size_type n, const std::string& val) {comments.assign(n, val);} - - // Related to the issue #97. - // - // It is known that `std::vector::insert` and `std::vector::erase` in - // the standard library implementation included in GCC 4.8.5 takes - // `std::vector::iterator` instead of `std::vector::const_iterator`. - // Because of the const-correctness, we cannot convert a `const_iterator` to - // an `iterator`. It causes compilation error in GCC 4.8.5. -#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) && !defined(__clang__) -# if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) <= 40805 -# define TOML11_WORKAROUND_GCC_4_8_X_STANDARD_LIBRARY_IMPLEMENTATION -# endif -#endif - -#ifdef TOML11_WORKAROUND_GCC_4_8_X_STANDARD_LIBRARY_IMPLEMENTATION - iterator insert(iterator p, const std::string& x) - { - return comments.insert(p, x); - } - iterator insert(iterator p, std::string&& x) - { - return comments.insert(p, std::move(x)); - } - void insert(iterator p, size_type n, const std::string& x) - { - return comments.insert(p, n, x); - } - template - void insert(iterator p, InputIterator first, InputIterator last) - { - return comments.insert(p, first, last); - } - void insert(iterator p, std::initializer_list ini) - { - return comments.insert(p, ini); - } - - template - iterator emplace(iterator p, Ts&& ... args) - { - return comments.emplace(p, std::forward(args)...); - } - - iterator erase(iterator pos) {return comments.erase(pos);} - iterator erase(iterator first, iterator last) - { - return comments.erase(first, last); - } -#else - iterator insert(const_iterator p, const std::string& x) - { - return comments.insert(p, x); - } - iterator insert(const_iterator p, std::string&& x) - { - return comments.insert(p, std::move(x)); - } - iterator insert(const_iterator p, size_type n, const std::string& x) - { - return comments.insert(p, n, x); - } - template - iterator insert(const_iterator p, InputIterator first, InputIterator last) - { - return comments.insert(p, first, last); - } - iterator insert(const_iterator p, std::initializer_list ini) - { - return comments.insert(p, ini); - } - - template - iterator emplace(const_iterator p, Ts&& ... args) - { - return comments.emplace(p, std::forward(args)...); - } - - iterator erase(const_iterator pos) {return comments.erase(pos);} - iterator erase(const_iterator first, const_iterator last) - { - return comments.erase(first, last); - } -#endif - - void swap(preserve_comments& other) {comments.swap(other.comments);} - - void push_back(const std::string& v) {comments.push_back(v);} - void push_back(std::string&& v) {comments.push_back(std::move(v));} - void pop_back() {comments.pop_back();} - - template - void emplace_back(Ts&& ... args) {comments.emplace_back(std::forward(args)...);} - - void clear() {comments.clear();} - - size_type size() const noexcept {return comments.size();} - size_type max_size() const noexcept {return comments.max_size();} - size_type capacity() const noexcept {return comments.capacity();} - bool empty() const noexcept {return comments.empty();} - - void reserve(size_type n) {comments.reserve(n);} - void resize(size_type n) {comments.resize(n);} - void resize(size_type n, const std::string& c) {comments.resize(n, c);} - void shrink_to_fit() {comments.shrink_to_fit();} - - reference operator[](const size_type n) noexcept {return comments[n];} - const_reference operator[](const size_type n) const noexcept {return comments[n];} - reference at(const size_type n) {return comments.at(n);} - const_reference at(const size_type n) const {return comments.at(n);} - reference front() noexcept {return comments.front();} - const_reference front() const noexcept {return comments.front();} - reference back() noexcept {return comments.back();} - const_reference back() const noexcept {return comments.back();} - - pointer data() noexcept {return comments.data();} - const_pointer data() const noexcept {return comments.data();} - - iterator begin() noexcept {return comments.begin();} - iterator end() noexcept {return comments.end();} - const_iterator begin() const noexcept {return comments.begin();} - const_iterator end() const noexcept {return comments.end();} - const_iterator cbegin() const noexcept {return comments.cbegin();} - const_iterator cend() const noexcept {return comments.cend();} - - reverse_iterator rbegin() noexcept {return comments.rbegin();} - reverse_iterator rend() noexcept {return comments.rend();} - const_reverse_iterator rbegin() const noexcept {return comments.rbegin();} - const_reverse_iterator rend() const noexcept {return comments.rend();} - const_reverse_iterator crbegin() const noexcept {return comments.crbegin();} - const_reverse_iterator crend() const noexcept {return comments.crend();} - - friend bool operator==(const preserve_comments&, const preserve_comments&); - friend bool operator!=(const preserve_comments&, const preserve_comments&); - friend bool operator< (const preserve_comments&, const preserve_comments&); - friend bool operator<=(const preserve_comments&, const preserve_comments&); - friend bool operator> (const preserve_comments&, const preserve_comments&); - friend bool operator>=(const preserve_comments&, const preserve_comments&); - - friend void swap(preserve_comments&, std::vector&); - friend void swap(std::vector&, preserve_comments&); - - private: - - container_type comments; -}; - -inline bool operator==(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments == rhs.comments;} -inline bool operator!=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments != rhs.comments;} -inline bool operator< (const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments < rhs.comments;} -inline bool operator<=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments <= rhs.comments;} -inline bool operator> (const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments > rhs.comments;} -inline bool operator>=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments >= rhs.comments;} - -inline void swap(preserve_comments& lhs, preserve_comments& rhs) -{ - lhs.swap(rhs); - return; -} -inline void swap(preserve_comments& lhs, std::vector& rhs) -{ - lhs.comments.swap(rhs); - return; -} -inline void swap(std::vector& lhs, preserve_comments& rhs) -{ - lhs.swap(rhs.comments); - return; -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const preserve_comments& com) -{ - for(const auto& c : com) - { - os << '#' << c << '\n'; - } - return os; -} - -namespace detail -{ - -// To provide the same interface with `preserve_comments`, `discard_comments` -// should have an iterator. But it does not contain anything, so we need to -// add an iterator that points nothing. -// -// It always points null, so DO NOT unwrap this iterator. It always crashes -// your program. -template -struct empty_iterator -{ - using value_type = T; - using reference_type = typename std::conditional::type; - using pointer_type = typename std::conditional::type; - using difference_type = std::ptrdiff_t; - using iterator_category = std::random_access_iterator_tag; - - empty_iterator() = default; - ~empty_iterator() = default; - empty_iterator(empty_iterator const&) = default; - empty_iterator(empty_iterator &&) = default; - empty_iterator& operator=(empty_iterator const&) = default; - empty_iterator& operator=(empty_iterator &&) = default; - - // DO NOT call these operators. - reference_type operator*() const noexcept {std::terminate();} - pointer_type operator->() const noexcept {return nullptr;} - reference_type operator[](difference_type) const noexcept {return this->operator*();} - - // These operators do nothing. - empty_iterator& operator++() noexcept {return *this;} - empty_iterator operator++(int) noexcept {return *this;} - empty_iterator& operator--() noexcept {return *this;} - empty_iterator operator--(int) noexcept {return *this;} - - empty_iterator& operator+=(difference_type) noexcept {return *this;} - empty_iterator& operator-=(difference_type) noexcept {return *this;} - - empty_iterator operator+(difference_type) const noexcept {return *this;} - empty_iterator operator-(difference_type) const noexcept {return *this;} -}; - -template -bool operator==(const empty_iterator&, const empty_iterator&) noexcept {return true;} -template -bool operator!=(const empty_iterator&, const empty_iterator&) noexcept {return false;} -template -bool operator< (const empty_iterator&, const empty_iterator&) noexcept {return false;} -template -bool operator<=(const empty_iterator&, const empty_iterator&) noexcept {return true;} -template -bool operator> (const empty_iterator&, const empty_iterator&) noexcept {return false;} -template -bool operator>=(const empty_iterator&, const empty_iterator&) noexcept {return true;} - -template -typename empty_iterator::difference_type -operator-(const empty_iterator&, const empty_iterator&) noexcept {return 0;} - -template -empty_iterator -operator+(typename empty_iterator::difference_type, const empty_iterator& rhs) noexcept {return rhs;} -template -empty_iterator -operator+(const empty_iterator& lhs, typename empty_iterator::difference_type) noexcept {return lhs;} - -} // detail - -// The default comment type. It discards all the comments. It requires only one -// byte to contain, so the memory footprint is smaller than preserve_comments. -// -// It just ignores `push_back`, `insert`, `erase`, and any other modifications. -// IT always returns size() == 0, the iterator taken by `begin()` is always the -// same as that of `end()`, and accessing through `operator[]` or iterators -// always causes a segmentation fault. DO NOT access to the element of this. -// -// Why this is chose as the default type is because the last version (2.x.y) -// does not contain any comments in a value. To minimize the impact on the -// efficiency, this is chosen as a default. -// -// To reduce the memory footprint, later we can try empty base optimization (EBO). -struct discard_comments -{ - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - using value_type = std::string; - using reference = std::string&; - using const_reference = std::string const&; - using pointer = std::string*; - using const_pointer = std::string const*; - using iterator = detail::empty_iterator; - using const_iterator = detail::empty_iterator; - using reverse_iterator = detail::empty_iterator; - using const_reverse_iterator = detail::empty_iterator; - - discard_comments() = default; - ~discard_comments() = default; - discard_comments(discard_comments const&) = default; - discard_comments(discard_comments &&) = default; - discard_comments& operator=(discard_comments const&) = default; - discard_comments& operator=(discard_comments &&) = default; - - explicit discard_comments(const std::vector&) noexcept {} - explicit discard_comments(std::vector&&) noexcept {} - discard_comments& operator=(const std::vector&) noexcept {return *this;} - discard_comments& operator=(std::vector&&) noexcept {return *this;} - - explicit discard_comments(const preserve_comments&) noexcept {} - - explicit discard_comments(size_type) noexcept {} - discard_comments(size_type, const std::string&) noexcept {} - discard_comments(std::initializer_list) noexcept {} - template - discard_comments(InputIterator, InputIterator) noexcept {} - - template - void assign(InputIterator, InputIterator) noexcept {} - void assign(std::initializer_list) noexcept {} - void assign(size_type, const std::string&) noexcept {} - - iterator insert(const_iterator, const std::string&) {return iterator{};} - iterator insert(const_iterator, std::string&&) {return iterator{};} - iterator insert(const_iterator, size_type, const std::string&) {return iterator{};} - template - iterator insert(const_iterator, InputIterator, InputIterator) {return iterator{};} - iterator insert(const_iterator, std::initializer_list) {return iterator{};} - - template - iterator emplace(const_iterator, Ts&& ...) {return iterator{};} - iterator erase(const_iterator) {return iterator{};} - iterator erase(const_iterator, const_iterator) {return iterator{};} - - void swap(discard_comments&) {return;} - - void push_back(const std::string&) {return;} - void push_back(std::string&& ) {return;} - void pop_back() {return;} - - template - void emplace_back(Ts&& ...) {return;} - - void clear() {return;} - - size_type size() const noexcept {return 0;} - size_type max_size() const noexcept {return 0;} - size_type capacity() const noexcept {return 0;} - bool empty() const noexcept {return true;} - - void reserve(size_type) {return;} - void resize(size_type) {return;} - void resize(size_type, const std::string&) {return;} - void shrink_to_fit() {return;} - - // DO NOT access to the element of this container. This container is always - // empty, so accessing through operator[], front/back, data causes address - // error. - - reference operator[](const size_type) noexcept {return *data();} - const_reference operator[](const size_type) const noexcept {return *data();} - reference at(const size_type) {throw std::out_of_range("toml::discard_comment is always empty.");} - const_reference at(const size_type) const {throw std::out_of_range("toml::discard_comment is always empty.");} - reference front() noexcept {return *data();} - const_reference front() const noexcept {return *data();} - reference back() noexcept {return *data();} - const_reference back() const noexcept {return *data();} - - pointer data() noexcept {return nullptr;} - const_pointer data() const noexcept {return nullptr;} - - iterator begin() noexcept {return iterator{};} - iterator end() noexcept {return iterator{};} - const_iterator begin() const noexcept {return const_iterator{};} - const_iterator end() const noexcept {return const_iterator{};} - const_iterator cbegin() const noexcept {return const_iterator{};} - const_iterator cend() const noexcept {return const_iterator{};} - - reverse_iterator rbegin() noexcept {return iterator{};} - reverse_iterator rend() noexcept {return iterator{};} - const_reverse_iterator rbegin() const noexcept {return const_iterator{};} - const_reverse_iterator rend() const noexcept {return const_iterator{};} - const_reverse_iterator crbegin() const noexcept {return const_iterator{};} - const_reverse_iterator crend() const noexcept {return const_iterator{};} -}; - -inline bool operator==(const discard_comments&, const discard_comments&) noexcept {return true;} -inline bool operator!=(const discard_comments&, const discard_comments&) noexcept {return false;} -inline bool operator< (const discard_comments&, const discard_comments&) noexcept {return false;} -inline bool operator<=(const discard_comments&, const discard_comments&) noexcept {return true;} -inline bool operator> (const discard_comments&, const discard_comments&) noexcept {return false;} -inline bool operator>=(const discard_comments&, const discard_comments&) noexcept {return true;} - -inline void swap(const discard_comments&, const discard_comments&) noexcept {return;} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const discard_comments&) -{ - return os; -} - -} // toml11 -#endif// TOML11_COMMENTS_HPP diff -Nru nix-2.8.0/src/toml11/toml/datetime.hpp nix-2.16.1+dfsg/src/toml11/toml/datetime.hpp --- nix-2.8.0/src/toml11/toml/datetime.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/datetime.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,631 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_DATETIME_HPP -#define TOML11_DATETIME_HPP -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace toml -{ - -// To avoid non-threadsafe std::localtime. In C11 (not C++11!), localtime_s is -// provided in the absolutely same purpose, but C++11 is actually not compatible -// with C11. We need to dispatch the function depending on the OS. -namespace detail -{ -// TODO: find more sophisticated way to handle this -#if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 1) || defined(_XOPEN_SOURCE) || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(_POSIX_SOURCE) -inline std::tm localtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::localtime_r(src, &dst); - if (!result) { throw std::runtime_error("localtime_r failed."); } - return dst; -} -inline std::tm gmtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::gmtime_r(src, &dst); - if (!result) { throw std::runtime_error("gmtime_r failed."); } - return dst; -} -#elif defined(_MSC_VER) -inline std::tm localtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::localtime_s(&dst, src); - if (result) { throw std::runtime_error("localtime_s failed."); } - return dst; -} -inline std::tm gmtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::gmtime_s(&dst, src); - if (result) { throw std::runtime_error("gmtime_s failed."); } - return dst; -} -#else // fallback. not threadsafe -inline std::tm localtime_s(const std::time_t* src) -{ - const auto result = std::localtime(src); - if (!result) { throw std::runtime_error("localtime failed."); } - return *result; -} -inline std::tm gmtime_s(const std::time_t* src) -{ - const auto result = std::gmtime(src); - if (!result) { throw std::runtime_error("gmtime failed."); } - return *result; -} -#endif -} // detail - -enum class month_t : std::uint8_t -{ - Jan = 0, - Feb = 1, - Mar = 2, - Apr = 3, - May = 4, - Jun = 5, - Jul = 6, - Aug = 7, - Sep = 8, - Oct = 9, - Nov = 10, - Dec = 11 -}; - -struct local_date -{ - std::int16_t year; // A.D. (like, 2018) - std::uint8_t month; // [0, 11] - std::uint8_t day; // [1, 31] - - local_date(int y, month_t m, int d) - : year (static_cast(y)), - month(static_cast(m)), - day (static_cast(d)) - {} - - explicit local_date(const std::tm& t) - : year (static_cast(t.tm_year + 1900)), - month(static_cast(t.tm_mon)), - day (static_cast(t.tm_mday)) - {} - - explicit local_date(const std::chrono::system_clock::time_point& tp) - { - const auto t = std::chrono::system_clock::to_time_t(tp); - const auto time = detail::localtime_s(&t); - *this = local_date(time); - } - - explicit local_date(const std::time_t t) - : local_date(std::chrono::system_clock::from_time_t(t)) - {} - - operator std::chrono::system_clock::time_point() const - { - // std::mktime returns date as local time zone. no conversion needed - std::tm t; - t.tm_sec = 0; - t.tm_min = 0; - t.tm_hour = 0; - t.tm_mday = static_cast(this->day); - t.tm_mon = static_cast(this->month); - t.tm_year = static_cast(this->year) - 1900; - t.tm_wday = 0; // the value will be ignored - t.tm_yday = 0; // the value will be ignored - t.tm_isdst = -1; - return std::chrono::system_clock::from_time_t(std::mktime(&t)); - } - - operator std::time_t() const - { - return std::chrono::system_clock::to_time_t( - std::chrono::system_clock::time_point(*this)); - } - - local_date() = default; - ~local_date() = default; - local_date(local_date const&) = default; - local_date(local_date&&) = default; - local_date& operator=(local_date const&) = default; - local_date& operator=(local_date&&) = default; -}; - -inline bool operator==(const local_date& lhs, const local_date& rhs) -{ - return std::make_tuple(lhs.year, lhs.month, lhs.day) == - std::make_tuple(rhs.year, rhs.month, rhs.day); -} -inline bool operator!=(const local_date& lhs, const local_date& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const local_date& lhs, const local_date& rhs) -{ - return std::make_tuple(lhs.year, lhs.month, lhs.day) < - std::make_tuple(rhs.year, rhs.month, rhs.day); -} -inline bool operator<=(const local_date& lhs, const local_date& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const local_date& lhs, const local_date& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const local_date& lhs, const local_date& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const local_date& date) -{ - os << std::setfill('0') << std::setw(4) << static_cast(date.year ) << '-'; - os << std::setfill('0') << std::setw(2) << static_cast(date.month) + 1 << '-'; - os << std::setfill('0') << std::setw(2) << static_cast(date.day ) ; - return os; -} - -struct local_time -{ - std::uint8_t hour; // [0, 23] - std::uint8_t minute; // [0, 59] - std::uint8_t second; // [0, 60] - std::uint16_t millisecond; // [0, 999] - std::uint16_t microsecond; // [0, 999] - std::uint16_t nanosecond; // [0, 999] - - local_time(int h, int m, int s, - int ms = 0, int us = 0, int ns = 0) - : hour (static_cast(h)), - minute(static_cast(m)), - second(static_cast(s)), - millisecond(static_cast(ms)), - microsecond(static_cast(us)), - nanosecond (static_cast(ns)) - {} - - explicit local_time(const std::tm& t) - : hour (static_cast(t.tm_hour)), - minute(static_cast(t.tm_min)), - second(static_cast(t.tm_sec)), - millisecond(0), microsecond(0), nanosecond(0) - {} - - template - explicit local_time(const std::chrono::duration& t) - { - const auto h = std::chrono::duration_cast(t); - this->hour = static_cast(h.count()); - const auto t2 = t - h; - const auto m = std::chrono::duration_cast(t2); - this->minute = static_cast(m.count()); - const auto t3 = t2 - m; - const auto s = std::chrono::duration_cast(t3); - this->second = static_cast(s.count()); - const auto t4 = t3 - s; - const auto ms = std::chrono::duration_cast(t4); - this->millisecond = static_cast(ms.count()); - const auto t5 = t4 - ms; - const auto us = std::chrono::duration_cast(t5); - this->microsecond = static_cast(us.count()); - const auto t6 = t5 - us; - const auto ns = std::chrono::duration_cast(t6); - this->nanosecond = static_cast(ns.count()); - } - - operator std::chrono::nanoseconds() const - { - return std::chrono::nanoseconds (this->nanosecond) + - std::chrono::microseconds(this->microsecond) + - std::chrono::milliseconds(this->millisecond) + - std::chrono::seconds(this->second) + - std::chrono::minutes(this->minute) + - std::chrono::hours(this->hour); - } - - local_time() = default; - ~local_time() = default; - local_time(local_time const&) = default; - local_time(local_time&&) = default; - local_time& operator=(local_time const&) = default; - local_time& operator=(local_time&&) = default; -}; - -inline bool operator==(const local_time& lhs, const local_time& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute, lhs.second, lhs.millisecond, lhs.microsecond, lhs.nanosecond) == - std::make_tuple(rhs.hour, rhs.minute, rhs.second, rhs.millisecond, rhs.microsecond, rhs.nanosecond); -} -inline bool operator!=(const local_time& lhs, const local_time& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const local_time& lhs, const local_time& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute, lhs.second, lhs.millisecond, lhs.microsecond, lhs.nanosecond) < - std::make_tuple(rhs.hour, rhs.minute, rhs.second, rhs.millisecond, rhs.microsecond, rhs.nanosecond); -} -inline bool operator<=(const local_time& lhs, const local_time& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const local_time& lhs, const local_time& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const local_time& lhs, const local_time& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const local_time& time) -{ - os << std::setfill('0') << std::setw(2) << static_cast(time.hour ) << ':'; - os << std::setfill('0') << std::setw(2) << static_cast(time.minute) << ':'; - os << std::setfill('0') << std::setw(2) << static_cast(time.second); - if(time.millisecond != 0 || time.microsecond != 0 || time.nanosecond != 0) - { - os << '.'; - os << std::setfill('0') << std::setw(3) << static_cast(time.millisecond); - if(time.microsecond != 0 || time.nanosecond != 0) - { - os << std::setfill('0') << std::setw(3) << static_cast(time.microsecond); - if(time.nanosecond != 0) - { - os << std::setfill('0') << std::setw(3) << static_cast(time.nanosecond); - } - } - } - return os; -} - -struct time_offset -{ - std::int8_t hour; // [-12, 12] - std::int8_t minute; // [-59, 59] - - time_offset(int h, int m) - : hour (static_cast(h)), - minute(static_cast(m)) - {} - - operator std::chrono::minutes() const - { - return std::chrono::minutes(this->minute) + - std::chrono::hours(this->hour); - } - - time_offset() = default; - ~time_offset() = default; - time_offset(time_offset const&) = default; - time_offset(time_offset&&) = default; - time_offset& operator=(time_offset const&) = default; - time_offset& operator=(time_offset&&) = default; -}; - -inline bool operator==(const time_offset& lhs, const time_offset& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute) == - std::make_tuple(rhs.hour, rhs.minute); -} -inline bool operator!=(const time_offset& lhs, const time_offset& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const time_offset& lhs, const time_offset& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute) < - std::make_tuple(rhs.hour, rhs.minute); -} -inline bool operator<=(const time_offset& lhs, const time_offset& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const time_offset& lhs, const time_offset& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const time_offset& lhs, const time_offset& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const time_offset& offset) -{ - if(offset.hour == 0 && offset.minute == 0) - { - os << 'Z'; - return os; - } - int minute = static_cast(offset.hour) * 60 + offset.minute; - if(minute < 0){os << '-'; minute = std::abs(minute);} else {os << '+';} - os << std::setfill('0') << std::setw(2) << minute / 60 << ':'; - os << std::setfill('0') << std::setw(2) << minute % 60; - return os; -} - -struct local_datetime -{ - local_date date; - local_time time; - - local_datetime(local_date d, local_time t): date(d), time(t) {} - - explicit local_datetime(const std::tm& t): date(t), time(t){} - - explicit local_datetime(const std::chrono::system_clock::time_point& tp) - { - const auto t = std::chrono::system_clock::to_time_t(tp); - std::tm ltime = detail::localtime_s(&t); - - this->date = local_date(ltime); - this->time = local_time(ltime); - - // std::tm lacks subsecond information, so diff between tp and tm - // can be used to get millisecond & microsecond information. - const auto t_diff = tp - - std::chrono::system_clock::from_time_t(std::mktime(<ime)); - this->time.millisecond = static_cast( - std::chrono::duration_cast(t_diff).count()); - this->time.microsecond = static_cast( - std::chrono::duration_cast(t_diff).count()); - this->time.nanosecond = static_cast( - std::chrono::duration_cast(t_diff).count()); - } - - explicit local_datetime(const std::time_t t) - : local_datetime(std::chrono::system_clock::from_time_t(t)) - {} - - operator std::chrono::system_clock::time_point() const - { - using internal_duration = - typename std::chrono::system_clock::time_point::duration; - - // Normally DST begins at A.M. 3 or 4. If we re-use conversion operator - // of local_date and local_time independently, the conversion fails if - // it is the day when DST begins or ends. Since local_date considers the - // time is 00:00 A.M. and local_time does not consider DST because it - // does not have any date information. We need to consider both date and - // time information at the same time to convert it correctly. - - std::tm t; - t.tm_sec = static_cast(this->time.second); - t.tm_min = static_cast(this->time.minute); - t.tm_hour = static_cast(this->time.hour); - t.tm_mday = static_cast(this->date.day); - t.tm_mon = static_cast(this->date.month); - t.tm_year = static_cast(this->date.year) - 1900; - t.tm_wday = 0; // the value will be ignored - t.tm_yday = 0; // the value will be ignored - t.tm_isdst = -1; - - // std::mktime returns date as local time zone. no conversion needed - auto dt = std::chrono::system_clock::from_time_t(std::mktime(&t)); - dt += std::chrono::duration_cast( - std::chrono::milliseconds(this->time.millisecond) + - std::chrono::microseconds(this->time.microsecond) + - std::chrono::nanoseconds (this->time.nanosecond)); - return dt; - } - - operator std::time_t() const - { - return std::chrono::system_clock::to_time_t( - std::chrono::system_clock::time_point(*this)); - } - - local_datetime() = default; - ~local_datetime() = default; - local_datetime(local_datetime const&) = default; - local_datetime(local_datetime&&) = default; - local_datetime& operator=(local_datetime const&) = default; - local_datetime& operator=(local_datetime&&) = default; -}; - -inline bool operator==(const local_datetime& lhs, const local_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time) == - std::make_tuple(rhs.date, rhs.time); -} -inline bool operator!=(const local_datetime& lhs, const local_datetime& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const local_datetime& lhs, const local_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time) < - std::make_tuple(rhs.date, rhs.time); -} -inline bool operator<=(const local_datetime& lhs, const local_datetime& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const local_datetime& lhs, const local_datetime& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const local_datetime& lhs, const local_datetime& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const local_datetime& dt) -{ - os << dt.date << 'T' << dt.time; - return os; -} - -struct offset_datetime -{ - local_date date; - local_time time; - time_offset offset; - - offset_datetime(local_date d, local_time t, time_offset o) - : date(d), time(t), offset(o) - {} - offset_datetime(const local_datetime& dt, time_offset o) - : date(dt.date), time(dt.time), offset(o) - {} - explicit offset_datetime(const local_datetime& ld) - : date(ld.date), time(ld.time), offset(get_local_offset(nullptr)) - // use the current local timezone offset - {} - explicit offset_datetime(const std::chrono::system_clock::time_point& tp) - : offset(0, 0) // use gmtime - { - const auto timet = std::chrono::system_clock::to_time_t(tp); - const auto tm = detail::gmtime_s(&timet); - this->date = local_date(tm); - this->time = local_time(tm); - } - explicit offset_datetime(const std::time_t& t) - : offset(0, 0) // use gmtime - { - const auto tm = detail::gmtime_s(&t); - this->date = local_date(tm); - this->time = local_time(tm); - } - explicit offset_datetime(const std::tm& t) - : offset(0, 0) // assume gmtime - { - this->date = local_date(t); - this->time = local_time(t); - } - - operator std::chrono::system_clock::time_point() const - { - // get date-time - using internal_duration = - typename std::chrono::system_clock::time_point::duration; - - // first, convert it to local date-time information in the same way as - // local_datetime does. later we will use time_t to adjust time offset. - std::tm t; - t.tm_sec = static_cast(this->time.second); - t.tm_min = static_cast(this->time.minute); - t.tm_hour = static_cast(this->time.hour); - t.tm_mday = static_cast(this->date.day); - t.tm_mon = static_cast(this->date.month); - t.tm_year = static_cast(this->date.year) - 1900; - t.tm_wday = 0; // the value will be ignored - t.tm_yday = 0; // the value will be ignored - t.tm_isdst = -1; - const std::time_t tp_loc = std::mktime(std::addressof(t)); - - auto tp = std::chrono::system_clock::from_time_t(tp_loc); - tp += std::chrono::duration_cast( - std::chrono::milliseconds(this->time.millisecond) + - std::chrono::microseconds(this->time.microsecond) + - std::chrono::nanoseconds (this->time.nanosecond)); - - // Since mktime uses local time zone, it should be corrected. - // `12:00:00+09:00` means `03:00:00Z`. So mktime returns `03:00:00Z` if - // we are in `+09:00` timezone. To represent `12:00:00Z` there, we need - // to add `+09:00` to `03:00:00Z`. - // Here, it uses the time_t converted from date-time info to handle - // daylight saving time. - const auto ofs = get_local_offset(std::addressof(tp_loc)); - tp += std::chrono::hours (ofs.hour); - tp += std::chrono::minutes(ofs.minute); - - // We got `12:00:00Z` by correcting local timezone applied by mktime. - // Then we will apply the offset. Let's say `12:00:00-08:00` is given. - // And now, we have `12:00:00Z`. `12:00:00-08:00` means `20:00:00Z`. - // So we need to subtract the offset. - tp -= std::chrono::minutes(this->offset); - return tp; - } - - operator std::time_t() const - { - return std::chrono::system_clock::to_time_t( - std::chrono::system_clock::time_point(*this)); - } - - offset_datetime() = default; - ~offset_datetime() = default; - offset_datetime(offset_datetime const&) = default; - offset_datetime(offset_datetime&&) = default; - offset_datetime& operator=(offset_datetime const&) = default; - offset_datetime& operator=(offset_datetime&&) = default; - - private: - - static time_offset get_local_offset(const std::time_t* tp) - { - // get local timezone with the same date-time information as mktime - const auto t = detail::localtime_s(tp); - - std::array buf; - const auto result = std::strftime(buf.data(), 6, "%z", &t); // +hhmm\0 - if(result != 5) - { - throw std::runtime_error("toml::offset_datetime: cannot obtain " - "timezone information of current env"); - } - const int ofs = std::atoi(buf.data()); - const int ofs_h = ofs / 100; - const int ofs_m = ofs - (ofs_h * 100); - return time_offset(ofs_h, ofs_m); - } -}; - -inline bool operator==(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time, lhs.offset) == - std::make_tuple(rhs.date, rhs.time, rhs.offset); -} -inline bool operator!=(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const offset_datetime& lhs, const offset_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time, lhs.offset) < - std::make_tuple(rhs.date, rhs.time, rhs.offset); -} -inline bool operator<=(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const offset_datetime& lhs, const offset_datetime& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const offset_datetime& dt) -{ - os << dt.date << 'T' << dt.time << dt.offset; - return os; -} - -}//toml -#endif// TOML11_DATETIME diff -Nru nix-2.8.0/src/toml11/toml/exception.hpp nix-2.16.1+dfsg/src/toml11/toml/exception.hpp --- nix-2.8.0/src/toml11/toml/exception.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/exception.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,65 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_EXCEPTION_HPP -#define TOML11_EXCEPTION_HPP -#include -#include - -#include "source_location.hpp" - -namespace toml -{ - -struct exception : public std::exception -{ - public: - explicit exception(const source_location& loc): loc_(loc) {} - virtual ~exception() noexcept override = default; - virtual const char* what() const noexcept override {return "";} - virtual source_location const& location() const noexcept {return loc_;} - - protected: - source_location loc_; -}; - -struct syntax_error : public toml::exception -{ - public: - explicit syntax_error(const std::string& what_arg, const source_location& loc) - : exception(loc), what_(what_arg) - {} - virtual ~syntax_error() noexcept override = default; - virtual const char* what() const noexcept override {return what_.c_str();} - - protected: - std::string what_; -}; - -struct type_error : public toml::exception -{ - public: - explicit type_error(const std::string& what_arg, const source_location& loc) - : exception(loc), what_(what_arg) - {} - virtual ~type_error() noexcept override = default; - virtual const char* what() const noexcept override {return what_.c_str();} - - protected: - std::string what_; -}; - -struct internal_error : public toml::exception -{ - public: - explicit internal_error(const std::string& what_arg, const source_location& loc) - : exception(loc), what_(what_arg) - {} - virtual ~internal_error() noexcept override = default; - virtual const char* what() const noexcept override {return what_.c_str();} - - protected: - std::string what_; -}; - -} // toml -#endif // TOML_EXCEPTION diff -Nru nix-2.8.0/src/toml11/toml/from.hpp nix-2.16.1+dfsg/src/toml11/toml/from.hpp --- nix-2.8.0/src/toml11/toml/from.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/from.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,19 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_FROM_HPP -#define TOML11_FROM_HPP - -namespace toml -{ - -template -struct from; -// { -// static T from_toml(const toml::value& v) -// { -// // User-defined conversions ... -// } -// }; - -} // toml -#endif // TOML11_FROM_HPP diff -Nru nix-2.8.0/src/toml11/toml/get.hpp nix-2.16.1+dfsg/src/toml11/toml/get.hpp --- nix-2.8.0/src/toml11/toml/get.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/get.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,1117 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_GET_HPP -#define TOML11_GET_HPP -#include - -#include "from.hpp" -#include "result.hpp" -#include "value.hpp" - -namespace toml -{ - -// ============================================================================ -// exact toml::* type - -template class M, template class V> -detail::enable_if_t>::value, T> & -get(basic_value& v) -{ - return v.template cast>::value>(); -} - -template class M, template class V> -detail::enable_if_t>::value, T> const& -get(const basic_value& v) -{ - return v.template cast>::value>(); -} - -template class M, template class V> -detail::enable_if_t>::value, T> -get(basic_value&& v) -{ - return T(std::move(v).template cast>::value>()); -} - -// ============================================================================ -// T == toml::value; identity transformation. - -template class M, template class V> -inline detail::enable_if_t>::value, T>& -get(basic_value& v) -{ - return v; -} - -template class M, template class V> -inline detail::enable_if_t>::value, T> const& -get(const basic_value& v) -{ - return v; -} - -template class M, template class V> -inline detail::enable_if_t>::value, T> -get(basic_value&& v) -{ - return basic_value(std::move(v)); -} - -// ============================================================================ -// T == toml::basic_value; basic_value -> basic_value - -template class M, template class V> -inline detail::enable_if_t, - detail::negation>> - >::value, T> -get(const basic_value& v) -{ - return T(v); -} - -// ============================================================================ -// integer convertible from toml::Integer - -template class M, template class V> -inline detail::enable_if_t, // T is integral - detail::negation>, // but not bool - detail::negation< // but not toml::integer - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - return static_cast(v.as_integer()); -} - -// ============================================================================ -// floating point convertible from toml::Float - -template class M, template class V> -inline detail::enable_if_t, // T is floating_point - detail::negation< // but not toml::floating - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - return static_cast(v.as_floating()); -} - -// ============================================================================ -// std::string; toml uses its own toml::string, but it should be convertible to -// std::string seamlessly - -template class M, template class V> -inline detail::enable_if_t::value, std::string>& -get(basic_value& v) -{ - return v.as_string().str; -} - -template class M, template class V> -inline detail::enable_if_t::value, std::string> const& -get(const basic_value& v) -{ - return v.as_string().str; -} - -template class M, template class V> -inline detail::enable_if_t::value, std::string> -get(basic_value&& v) -{ - return std::string(std::move(v.as_string().str)); -} - -// ============================================================================ -// std::string_view - -#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 -template class M, template class V> -inline detail::enable_if_t::value, std::string_view> -get(const basic_value& v) -{ - return std::string_view(v.as_string().str); -} -#endif - -// ============================================================================ -// std::chrono::duration from toml::local_time. - -template class M, template class V> -inline detail::enable_if_t::value, T> -get(const basic_value& v) -{ - return std::chrono::duration_cast( - std::chrono::nanoseconds(v.as_local_time())); -} - -// ============================================================================ -// std::chrono::system_clock::time_point from toml::datetime variants - -template class M, template class V> -inline detail::enable_if_t< - std::is_same::value, T> -get(const basic_value& v) -{ - switch(v.type()) - { - case value_t::local_date: - { - return std::chrono::system_clock::time_point(v.as_local_date()); - } - case value_t::local_datetime: - { - return std::chrono::system_clock::time_point(v.as_local_datetime()); - } - case value_t::offset_datetime: - { - return std::chrono::system_clock::time_point(v.as_offset_datetime()); - } - default: - { - throw type_error(detail::format_underline("toml::value: " - "bad_cast to std::chrono::system_clock::time_point", { - {v.location(), concat_to_string("the actual type is ", v.type())} - }), v.location()); - } - } -} - -// ============================================================================ -// forward declaration to use this recursively. ignore this and go ahead. - -// array-like type with push_back(value) method -template class M, template class V> -detail::enable_if_t, // T is a container - detail::has_push_back_method, // T::push_back(value) works - detail::negation< // but not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value&); - -// array-like type without push_back(value) method -template class M, template class V> -detail::enable_if_t, // T is a container - detail::negation>, // w/o push_back(...) - detail::negation< // not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value&); - -// std::pair -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value&); - -// std::tuple -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value&); - -// map-like classes -template class M, template class V> -detail::enable_if_t, // T is map - detail::negation< // but not toml::table - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value&); - -// T.from_toml(v) -template class M, template class V> -detail::enable_if_t>>, - detail::has_from_toml_method, // but has from_toml(toml::value) - std::is_default_constructible // and default constructible - >::value, T> -get(const basic_value&); - -// toml::from::from_toml(v) -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value&); - -// T(const toml::value&) and T is not toml::basic_value, -// and it does not have `from` nor `from_toml`. -template class M, template class V> -detail::enable_if_t>, - std::is_constructible&>, - detail::negation>, - detail::negation> - >::value, T> -get(const basic_value&); - -// ============================================================================ -// array-like types; most likely STL container, like std::vector, etc. - -template class M, template class V> -detail::enable_if_t, // T is a container - detail::has_push_back_method, // container.push_back(elem) works - detail::negation< // but not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - using value_type = typename T::value_type; - const auto& ary = v.as_array(); - - T container; - try_reserve(container, ary.size()); - - for(const auto& elem : ary) - { - container.push_back(get(elem)); - } - return container; -} - -// ============================================================================ -// std::forward_list does not have push_back, insert, or emplace. -// It has insert_after, emplace_after, push_front. - -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - using value_type = typename T::value_type; - T container; - for(const auto& elem : v.as_array()) - { - container.push_front(get(elem)); - } - container.reverse(); - return container; -} - -// ============================================================================ -// array-like types, without push_back(). most likely [std|boost]::array. - -template class M, template class V> -detail::enable_if_t, // T is a container - detail::negation>, // w/o push_back - detail::negation< // T is not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - using value_type = typename T::value_type; - const auto& ar = v.as_array(); - - T container; - if(ar.size() != container.size()) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "toml::get: specified container size is ", container.size(), - " but there are ", ar.size(), " elements in toml array."), { - {v.location(), "here"} - })); - } - for(std::size_t i=0; i(ar[i]); - } - return container; -} - -// ============================================================================ -// std::pair. - -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - using first_type = typename T::first_type; - using second_type = typename T::second_type; - - const auto& ar = v.as_array(); - if(ar.size() != 2) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "toml::get: specified std::pair but there are ", ar.size(), - " elements in toml array."), {{v.location(), "here"}})); - } - return std::make_pair(::toml::get(ar.at(0)), - ::toml::get(ar.at(1))); -} - -// ============================================================================ -// std::tuple. - -namespace detail -{ -template -T get_tuple_impl(const Array& a, index_sequence) -{ - return std::make_tuple( - ::toml::get::type>(a.at(I))...); -} -} // detail - -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - const auto& ar = v.as_array(); - if(ar.size() != std::tuple_size::value) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "toml::get: specified std::tuple with ", - std::tuple_size::value, " elements, but there are ", ar.size(), - " elements in toml array."), {{v.location(), "here"}})); - } - return detail::get_tuple_impl(ar, - detail::make_index_sequence::value>{}); -} - -// ============================================================================ -// map-like types; most likely STL map, like std::map or std::unordered_map. - -template class M, template class V> -detail::enable_if_t, // T is map - detail::negation< // but not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - using key_type = typename T::key_type; - using mapped_type = typename T::mapped_type; - static_assert(std::is_convertible::value, - "toml::get only supports map type of which key_type is " - "convertible from std::string."); - T map; - for(const auto& kv : v.as_table()) - { - map.emplace(key_type(kv.first), get(kv.second)); - } - return map; -} - -// ============================================================================ -// user-defined, but compatible types. - -template class M, template class V> -detail::enable_if_t>>, - detail::has_from_toml_method, // but has from_toml(toml::value) memfn - std::is_default_constructible // and default constructible - >::value, T> -get(const basic_value& v) -{ - T ud; - ud.from_toml(v); - return ud; -} -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - return ::toml::from::from_toml(v); -} - -template class M, template class V> -detail::enable_if_t>, // T is not a toml::value - std::is_constructible&>, // T is constructible from toml::value - detail::negation>, // and T does not have T.from_toml(v); - detail::negation> // and T does not have toml::from{}; - >::value, T> -get(const basic_value& v) -{ - return T(v); -} - -// ============================================================================ -// find - -// ---------------------------------------------------------------------------- -// these overloads do not require to set T. and returns value itself. -template class M, template class V> -basic_value const& find(const basic_value& v, const key& ky) -{ - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return tab.at(ky); -} -template class M, template class V> -basic_value& find(basic_value& v, const key& ky) -{ - auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return tab.at(ky); -} -template class M, template class V> -basic_value find(basic_value&& v, const key& ky) -{ - typename basic_value::table_type tab = std::move(v).as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return basic_value(std::move(tab.at(ky))); -} - -// ---------------------------------------------------------------------------- -// find(value, idx) -template class M, template class V> -basic_value const& -find(const basic_value& v, const std::size_t idx) -{ - const auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ary.at(idx); -} -template class M, template class V> -basic_value& find(basic_value& v, const std::size_t idx) -{ - auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ary.at(idx); -} -template class M, template class V> -basic_value find(basic_value&& v, const std::size_t idx) -{ - auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return basic_value(std::move(ary.at(idx))); -} - -// ---------------------------------------------------------------------------- -// find(value, key); - -template class M, template class V> -decltype(::toml::get(std::declval const&>())) -find(const basic_value& v, const key& ky) -{ - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return ::toml::get(tab.at(ky)); -} - -template class M, template class V> -decltype(::toml::get(std::declval&>())) -find(basic_value& v, const key& ky) -{ - auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return ::toml::get(tab.at(ky)); -} - -template class M, template class V> -decltype(::toml::get(std::declval&&>())) -find(basic_value&& v, const key& ky) -{ - typename basic_value::table_type tab = std::move(v).as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return ::toml::get(std::move(tab.at(ky))); -} - -// ---------------------------------------------------------------------------- -// find(value, idx) -template class M, template class V> -decltype(::toml::get(std::declval const&>())) -find(const basic_value& v, const std::size_t idx) -{ - const auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ::toml::get(ary.at(idx)); -} -template class M, template class V> -decltype(::toml::get(std::declval&>())) -find(basic_value& v, const std::size_t idx) -{ - auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ::toml::get(ary.at(idx)); -} -template class M, template class V> -decltype(::toml::get(std::declval&&>())) -find(basic_value&& v, const std::size_t idx) -{ - typename basic_value::array_type ary = std::move(v).as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ::toml::get(std::move(ary.at(idx))); -} - -// -------------------------------------------------------------------------- -// toml::find(toml::value, toml::key, Ts&& ... keys) - -namespace detail -{ -// It suppresses warnings by -Wsign-conversion. Let's say we have the following -// code. -// ```cpp -// const auto x = toml::find(data, "array", 0); -// ``` -// Here, the type of literal number `0` is `int`. `int` is a signed integer. -// `toml::find` takes `std::size_t` as an index. So it causes implicit sign -// conversion and `-Wsign-conversion` warns about it. Using `0u` instead of `0` -// suppresses the warning, but it makes user code messy. -// To suppress this warning, we need to be aware of type conversion caused -// by `toml::find(v, key1, key2, ... keys)`. But the thing is that the types of -// keys can be any combination of {string-like, size_t-like}. Of course we can't -// write down all the combinations. Thus we need to use some function that -// recognize the type of argument and cast it into `std::string` or -// `std::size_t` depending on the context. -// `key_cast` does the job. It has 2 overloads. One is invoked when the -// argument type is an integer and cast the argument into `std::size_t`. The -// other is invoked when the argument type is not an integer, possibly one of -// std::string, const char[N] or const char*, and construct std::string from -// the argument. -// `toml::find(v, k1, k2, ... ks)` uses `key_cast` before passing `ks` to -// `toml::find(v, k)` to suppress -Wsign-conversion. - -template -enable_if_t>, - negation, bool>>>::value, std::size_t> -key_cast(T&& v) noexcept -{ - return std::size_t(v); -} -template -enable_if_t>, - negation, bool>>>>::value, std::string> -key_cast(T&& v) noexcept -{ - return std::string(std::forward(v)); -} -} // detail - -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -const basic_value& -find(const basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -basic_value& -find(basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -basic_value -find(basic_value&& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(std::move(v), std::forward(k1)), - detail::key_cast(k2), std::forward(keys)...); -} - -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -decltype(::toml::get(std::declval&>())) -find(const basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -decltype(::toml::get(std::declval&>())) -find(basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -decltype(::toml::get(std::declval&&>())) -find(basic_value&& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(std::move(v), detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} - -// ============================================================================ -// get_or(value, fallback) - -template class M, template class V> -basic_value const& -get_or(const basic_value& v, const basic_value&) -{ - return v; -} -template class M, template class V> -basic_value& -get_or(basic_value& v, basic_value&) -{ - return v; -} -template class M, template class V> -basic_value -get_or(basic_value&& v, basic_value&&) -{ - return v; -} - -// ---------------------------------------------------------------------------- -// specialization for the exact toml types (return type becomes lvalue ref) - -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T> const& -get_or(const basic_value& v, const T& opt) -{ - try - { - return get>(v); - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T>& -get_or(basic_value& v, T& opt) -{ - try - { - return get>(v); - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t, - basic_value>::value, detail::remove_cvref_t> -get_or(basic_value&& v, T&& opt) -{ - try - { - return get>(std::move(v)); - } - catch(...) - { - return detail::remove_cvref_t(std::forward(opt)); - } -} - -// ---------------------------------------------------------------------------- -// specialization for std::string (return type becomes lvalue ref) - -template class M, template class V> -detail::enable_if_t, std::string>::value, - std::string> const& -get_or(const basic_value& v, const T& opt) -{ - try - { - return v.as_string().str; - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t::value, std::string>& -get_or(basic_value& v, T& opt) -{ - try - { - return v.as_string().str; - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t< - std::is_same, std::string>::value, std::string> -get_or(basic_value&& v, T&& opt) -{ - try - { - return std::move(v.as_string().str); - } - catch(...) - { - return std::string(std::forward(opt)); - } -} - -// ---------------------------------------------------------------------------- -// specialization for string literal - -template class M, template class V> -detail::enable_if_t::type>::value, std::string> -get_or(const basic_value& v, T&& opt) -{ - try - { - return std::move(v.as_string().str); - } - catch(...) - { - return std::string(std::forward(opt)); - } -} - -// ---------------------------------------------------------------------------- -// others (require type conversion and return type cannot be lvalue reference) - -template class M, template class V> -detail::enable_if_t, - basic_value>>, - detail::negation>>, - detail::negation::type>> - >::value, detail::remove_cvref_t> -get_or(const basic_value& v, T&& opt) -{ - try - { - return get>(v); - } - catch(...) - { - return detail::remove_cvref_t(std::forward(opt)); - } -} - -// =========================================================================== -// find_or(value, key, fallback) - -template class M, template class V> -basic_value const& -find_or(const basic_value& v, const key& ky, - const basic_value& opt) -{ - if(!v.is_table()) {return opt;} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return tab.at(ky); -} - -template class M, template class V> -basic_value& -find_or(basic_value& v, const toml::key& ky, basic_value& opt) -{ - if(!v.is_table()) {return opt;} - auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return tab.at(ky); -} - -template class M, template class V> -basic_value -find_or(basic_value&& v, const toml::key& ky, basic_value&& opt) -{ - if(!v.is_table()) {return opt;} - auto tab = std::move(v).as_table(); - if(tab.count(ky) == 0) {return opt;} - return basic_value(std::move(tab.at(ky))); -} - -// --------------------------------------------------------------------------- -// exact types (return type can be a reference) -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T> const& -find_or(const basic_value& v, const key& ky, const T& opt) -{ - if(!v.is_table()) {return opt;} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} - -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T>& -find_or(basic_value& v, const toml::key& ky, T& opt) -{ - if(!v.is_table()) {return opt;} - auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} - -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, - detail::remove_cvref_t> -find_or(basic_value&& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::forward(opt);} - auto tab = std::move(v).as_table(); - if(tab.count(ky) == 0) {return std::forward(opt);} - return get_or(std::move(tab.at(ky)), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// std::string (return type can be a reference) - -template class M, template class V> -detail::enable_if_t::value, std::string> const& -find_or(const basic_value& v, const key& ky, const T& opt) -{ - if(!v.is_table()) {return opt;} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} -template class M, template class V> -detail::enable_if_t::value, std::string>& -find_or(basic_value& v, const toml::key& ky, T& opt) -{ - if(!v.is_table()) {return opt;} - auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} -template class M, template class V> -detail::enable_if_t::value, std::string> -find_or(basic_value&& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::forward(opt);} - auto tab = std::move(v).as_table(); - if(tab.count(ky) == 0) {return std::forward(opt);} - return get_or(std::move(tab.at(ky)), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// string literal (deduced as std::string) -template class M, template class V> -detail::enable_if_t< - detail::is_string_literal::type>::value, - std::string> -find_or(const basic_value& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::string(opt);} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return std::string(opt);} - return get_or(tab.at(ky), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// others (require type conversion and return type cannot be lvalue reference) -template class M, template class V> -detail::enable_if_t, basic_value>>, - // T is not std::string - detail::negation>>, - // T is not a string literal - detail::negation::type>> - >::value, detail::remove_cvref_t> -find_or(const basic_value& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::forward(opt);} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return std::forward(opt);} - return get_or(tab.at(ky), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// recursive find-or with type deduction (find_or(value, keys, opt)) - -template 1), std::nullptr_t> = nullptr> - // here we need to add SFINAE in the template parameter to avoid - // infinite recursion in type deduction on gcc -auto find_or(Value&& v, const toml::key& ky, Ks&& ... keys) - -> decltype(find_or(std::forward(v), ky, detail::last_one(std::forward(keys)...))) -{ - if(!v.is_table()) - { - return detail::last_one(std::forward(keys)...); - } - auto&& tab = std::forward(v).as_table(); - if(tab.count(ky) == 0) - { - return detail::last_one(std::forward(keys)...); - } - return find_or(std::forward(tab).at(ky), std::forward(keys)...); -} - -// --------------------------------------------------------------------------- -// recursive find_or with explicit type specialization, find_or(value, keys...) - -template 1), std::nullptr_t> = nullptr> - // here we need to add SFINAE in the template parameter to avoid - // infinite recursion in type deduction on gcc -auto find_or(Value&& v, const toml::key& ky, Ks&& ... keys) - -> decltype(find_or(std::forward(v), ky, detail::last_one(std::forward(keys)...))) -{ - if(!v.is_table()) - { - return detail::last_one(std::forward(keys)...); - } - auto&& tab = std::forward(v).as_table(); - if(tab.count(ky) == 0) - { - return detail::last_one(std::forward(keys)...); - } - return find_or(std::forward(tab).at(ky), std::forward(keys)...); -} - -// ============================================================================ -// expect - -template class M, template class V> -result expect(const basic_value& v) noexcept -{ - try - { - return ok(get(v)); - } - catch(const std::exception& e) - { - return err(e.what()); - } -} -template class M, template class V> -result -expect(const basic_value& v, const toml::key& k) noexcept -{ - try - { - return ok(find(v, k)); - } - catch(const std::exception& e) - { - return err(e.what()); - } -} - -} // toml -#endif// TOML11_GET diff -Nru nix-2.8.0/src/toml11/toml/into.hpp nix-2.16.1+dfsg/src/toml11/toml/into.hpp --- nix-2.8.0/src/toml11/toml/into.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/into.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,19 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_INTO_HPP -#define TOML11_INTO_HPP - -namespace toml -{ - -template -struct into; -// { -// static toml::value into_toml(const T& user_defined_type) -// { -// // User-defined conversions ... -// } -// }; - -} // toml -#endif // TOML11_INTO_HPP diff -Nru nix-2.8.0/src/toml11/toml/lexer.hpp nix-2.16.1+dfsg/src/toml11/toml/lexer.hpp --- nix-2.8.0/src/toml11/toml/lexer.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/lexer.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,293 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_LEXER_HPP -#define TOML11_LEXER_HPP -#include -#include -#include -#include - -#include "combinator.hpp" - -namespace toml -{ -namespace detail -{ - -// these scans contents from current location in a container of char -// and extract a region that matches their own pattern. -// to see the implementation of each component, see combinator.hpp. - -using lex_wschar = either, character<'\t'>>; -using lex_ws = repeat>; -using lex_newline = either, - sequence, character<'\n'>>>; -using lex_lower = in_range<'a', 'z'>; -using lex_upper = in_range<'A', 'Z'>; -using lex_alpha = either; -using lex_digit = in_range<'0', '9'>; -using lex_nonzero = in_range<'1', '9'>; -using lex_oct_dig = in_range<'0', '7'>; -using lex_bin_dig = in_range<'0', '1'>; -using lex_hex_dig = either, in_range<'a', 'f'>>; - -using lex_hex_prefix = sequence, character<'x'>>; -using lex_oct_prefix = sequence, character<'o'>>; -using lex_bin_prefix = sequence, character<'b'>>; -using lex_underscore = character<'_'>; -using lex_plus = character<'+'>; -using lex_minus = character<'-'>; -using lex_sign = either; - -// digit | nonzero 1*(digit | _ digit) -using lex_unsigned_dec_int = either>, at_least<1>>>, - lex_digit>; -// (+|-)? unsigned_dec_int -using lex_dec_int = sequence, lex_unsigned_dec_int>; - -// hex_prefix hex_dig *(hex_dig | _ hex_dig) -using lex_hex_int = sequence>, unlimited>>>; -// oct_prefix oct_dig *(oct_dig | _ oct_dig) -using lex_oct_int = sequence>, unlimited>>>; -// bin_prefix bin_dig *(bin_dig | _ bin_dig) -using lex_bin_int = sequence>, unlimited>>>; - -// (dec_int | hex_int | oct_int | bin_int) -using lex_integer = either; - -// =========================================================================== - -using lex_inf = sequence, character<'n'>, character<'f'>>; -using lex_nan = sequence, character<'a'>, character<'n'>>; -using lex_special_float = sequence, either>; - -using lex_zero_prefixable_int = sequence>, unlimited>>; - -using lex_fractional_part = sequence, lex_zero_prefixable_int>; - -using lex_exponent_part = sequence, character<'E'>>, - maybe, lex_zero_prefixable_int>; - -using lex_float = either>>>>; - -// =========================================================================== - -using lex_true = sequence, character<'r'>, - character<'u'>, character<'e'>>; -using lex_false = sequence, character<'a'>, character<'l'>, - character<'s'>, character<'e'>>; -using lex_boolean = either; - -// =========================================================================== - -using lex_date_fullyear = repeat>; -using lex_date_month = repeat>; -using lex_date_mday = repeat>; -using lex_time_delim = either, character<'t'>, character<' '>>; -using lex_time_hour = repeat>; -using lex_time_minute = repeat>; -using lex_time_second = repeat>; -using lex_time_secfrac = sequence, - repeat>>; - -using lex_time_numoffset = sequence, character<'-'>>, - sequence, - lex_time_minute>>; -using lex_time_offset = either, character<'z'>, - lex_time_numoffset>; - -using lex_partial_time = sequence, - lex_time_minute, character<':'>, - lex_time_second, maybe>; -using lex_full_date = sequence, - lex_date_month, character<'-'>, - lex_date_mday>; -using lex_full_time = sequence; - -using lex_offset_date_time = sequence; -using lex_local_date_time = sequence; -using lex_local_date = lex_full_date; -using lex_local_time = lex_partial_time; - -// =========================================================================== - -using lex_quotation_mark = character<'"'>; -using lex_basic_unescaped = exclude, // 0x09 (tab) is allowed - in_range<0x0A, 0x1F>, - character<0x22>, character<0x5C>, - character<0x7F>>>; - -using lex_escape = character<'\\'>; -using lex_escape_unicode_short = sequence, - repeat>>; -using lex_escape_unicode_long = sequence, - repeat>>; -using lex_escape_seq_char = either, character<'\\'>, - character<'b'>, character<'f'>, - character<'n'>, character<'r'>, - character<'t'>, - lex_escape_unicode_short, - lex_escape_unicode_long - >; -using lex_escaped = sequence; -using lex_basic_char = either; -using lex_basic_string = sequence, - lex_quotation_mark>; - -// After toml post-v0.5.0, it is explicitly clarified how quotes in ml-strings -// are allowed to be used. -// After this, the following strings are *explicitly* allowed. -// - One or two `"`s in a multi-line basic string is allowed wherever it is. -// - Three consecutive `"`s in a multi-line basic string is considered as a delimiter. -// - One or two `"`s can appear just before or after the delimiter. -// ```toml -// str4 = """Here are two quotation marks: "". Simple enough.""" -// str5 = """Here are three quotation marks: ""\".""" -// str6 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\".""" -// str7 = """"This," she said, "is just a pointless statement."""" -// ``` -// In the current implementation (v3.3.0), it is difficult to parse `str7` in -// the above example. It is difficult to recognize `"` at the end of string body -// collectly. It will be misunderstood as a `"""` delimiter and an additional, -// invalid `"`. Like this: -// ```console -// what(): [error] toml::parse_table: invalid line format -// --> hoge.toml -// | -// 13 | str7 = """"This," she said, "is just a pointless statement."""" -// | ^- expected newline, but got '"'. -// ``` -// As a quick workaround for this problem, `lex_ml_basic_string_delim` was -// split into two, `lex_ml_basic_string_open` and `lex_ml_basic_string_close`. -// `lex_ml_basic_string_open` allows only `"""`. `_close` allows 3-5 `"`s. -// In parse_ml_basic_string() function, the trailing `"`s will be attached to -// the string body. -// -using lex_ml_basic_string_delim = repeat>; -using lex_ml_basic_string_open = lex_ml_basic_string_delim; -using lex_ml_basic_string_close = sequence< - repeat>, - maybe, maybe - >; - -using lex_ml_basic_unescaped = exclude, // 0x09 is tab - in_range<0x0A, 0x1F>, - character<0x5C>, // backslash - character<0x7F>, // DEL - lex_ml_basic_string_delim>>; - -using lex_ml_basic_escaped_newline = sequence< - lex_escape, maybe, lex_newline, - repeat, unlimited>>; - -using lex_ml_basic_char = either; -using lex_ml_basic_body = repeat, - unlimited>; -using lex_ml_basic_string = sequence; - -using lex_literal_char = exclude, in_range<0x0A, 0x1F>, - character<0x7F>, character<0x27>>>; -using lex_apostrophe = character<'\''>; -using lex_literal_string = sequence, - lex_apostrophe>; - -// the same reason as above. -using lex_ml_literal_string_delim = repeat>; -using lex_ml_literal_string_open = lex_ml_literal_string_delim; -using lex_ml_literal_string_close = sequence< - repeat>, - maybe, maybe - >; - -using lex_ml_literal_char = exclude, - in_range<0x0A, 0x1F>, - character<0x7F>, - lex_ml_literal_string_delim>>; -using lex_ml_literal_body = repeat, - unlimited>; -using lex_ml_literal_string = sequence; - -using lex_string = either; - -// =========================================================================== -using lex_dot_sep = sequence, character<'.'>, maybe>; - -using lex_unquoted_key = repeat, character<'_'>>, - at_least<1>>; -using lex_quoted_key = either; -using lex_simple_key = either; -using lex_dotted_key = sequence, - at_least<1> - > - >; -using lex_key = either; - -using lex_keyval_sep = sequence, - character<'='>, - maybe>; - -using lex_std_table_open = character<'['>; -using lex_std_table_close = character<']'>; -using lex_std_table = sequence, - lex_key, - maybe, - lex_std_table_close>; - -using lex_array_table_open = sequence; -using lex_array_table_close = sequence; -using lex_array_table = sequence, - lex_key, - maybe, - lex_array_table_close>; - -using lex_utf8_1byte = in_range<0x00, 0x7F>; -using lex_utf8_2byte = sequence< - in_range(0xC2), static_cast(0xDF)>, - in_range(0x80), static_cast(0xBF)> - >; -using lex_utf8_3byte = sequence(0xE0)>, in_range(0xA0), static_cast(0xBF)>>, - sequence(0xE1), static_cast(0xEC)>, in_range(0x80), static_cast(0xBF)>>, - sequence(0xED)>, in_range(0x80), static_cast(0x9F)>>, - sequence(0xEE), static_cast(0xEF)>, in_range(0x80), static_cast(0xBF)>> - >, in_range(0x80), static_cast(0xBF)>>; -using lex_utf8_4byte = sequence(0xF0)>, in_range(0x90), static_cast(0xBF)>>, - sequence(0xF1), static_cast(0xF3)>, in_range(0x80), static_cast(0xBF)>>, - sequence(0xF4)>, in_range(0x80), static_cast(0x8F)>> - >, in_range(0x80), static_cast(0xBF)>, - in_range(0x80), static_cast(0xBF)>>; -using lex_utf8_code = either< - lex_utf8_1byte, - lex_utf8_2byte, - lex_utf8_3byte, - lex_utf8_4byte - >; - -using lex_comment_start_symbol = character<'#'>; -using lex_non_eol_ascii = either, in_range<0x20, 0x7E>>; -using lex_comment = sequence, unlimited>>; - -} // detail -} // toml -#endif // TOML_LEXER_HPP diff -Nru nix-2.8.0/src/toml11/toml/literal.hpp nix-2.16.1+dfsg/src/toml11/toml/literal.hpp --- nix-2.8.0/src/toml11/toml/literal.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/literal.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,113 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_LITERAL_HPP -#define TOML11_LITERAL_HPP -#include "parser.hpp" - -namespace toml -{ -inline namespace literals -{ -inline namespace toml_literals -{ - -// implementation -inline ::toml::basic_value -literal_internal_impl(::toml::detail::location loc) -{ - using value_type = ::toml::basic_value< - TOML11_DEFAULT_COMMENT_STRATEGY, std::unordered_map, std::vector>; - // if there are some comments or empty lines, skip them. - using skip_line = ::toml::detail::repeat, - ::toml::detail::maybe<::toml::detail::lex_comment>, - ::toml::detail::lex_newline - >, ::toml::detail::at_least<1>>; - skip_line::invoke(loc); - - // if there are some whitespaces before a value, skip them. - using skip_ws = ::toml::detail::repeat< - ::toml::detail::lex_ws, ::toml::detail::at_least<1>>; - skip_ws::invoke(loc); - - // to distinguish arrays and tables, first check it is a table or not. - // - // "[1,2,3]"_toml; // this is an array - // "[table]"_toml; // a table that has an empty table named "table" inside. - // "[[1,2,3]]"_toml; // this is an array of arrays - // "[[table]]"_toml; // this is a table that has an array of tables inside. - // - // "[[1]]"_toml; // this can be both... (currently it becomes a table) - // "1 = [{}]"_toml; // this is a table that has an array of table named 1. - // "[[1,]]"_toml; // this is an array of arrays. - // "[[1],]"_toml; // this also. - - const auto the_front = loc.iter(); - - const bool is_table_key = ::toml::detail::lex_std_table::invoke(loc); - loc.reset(the_front); - - const bool is_aots_key = ::toml::detail::lex_array_table::invoke(loc); - loc.reset(the_front); - - // If it is neither a table-key or a array-of-table-key, it may be a value. - if(!is_table_key && !is_aots_key) - { - if(auto data = ::toml::detail::parse_value(loc)) - { - return data.unwrap(); - } - } - - // Note that still it can be a table, because the literal might be something - // like the following. - // ```cpp - // R"( // c++11 raw string literals - // key = "value" - // int = 42 - // )"_toml; - // ``` - // It is a valid toml file. - // It should be parsed as if we parse a file with this content. - - if(auto data = ::toml::detail::parse_toml_file(loc)) - { - return data.unwrap(); - } - else // none of them. - { - throw ::toml::syntax_error(data.unwrap_err(), source_location(loc)); - } - -} - -inline ::toml::basic_value -operator"" _toml(const char* str, std::size_t len) -{ - ::toml::detail::location loc( - std::string("TOML literal encoded in a C++ code"), - std::vector(str, str + len)); - // literal length does not include the null character at the end. - return literal_internal_impl(std::move(loc)); -} - -// value of __cplusplus in C++2a/20 mode is not fixed yet along compilers. -// So here we use the feature test macro for `char8_t` itself. -#if defined(__cpp_char8_t) && __cpp_char8_t >= 201811L -// value of u8"" literal has been changed from char to char8_t and char8_t is -// NOT compatible to char -inline ::toml::basic_value -operator"" _toml(const char8_t* str, std::size_t len) -{ - ::toml::detail::location loc( - std::string("TOML literal encoded in a C++ code"), - std::vector(reinterpret_cast(str), - reinterpret_cast(str) + len)); - return literal_internal_impl(std::move(loc)); -} -#endif - -} // toml_literals -} // literals -} // toml -#endif//TOML11_LITERAL_HPP diff -Nru nix-2.8.0/src/toml11/toml/macros.hpp nix-2.16.1+dfsg/src/toml11/toml/macros.hpp --- nix-2.8.0/src/toml11/toml/macros.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/macros.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,121 +0,0 @@ -#ifndef TOML11_MACROS_HPP -#define TOML11_MACROS_HPP - -#define TOML11_STRINGIZE_AUX(x) #x -#define TOML11_STRINGIZE(x) TOML11_STRINGIZE_AUX(x) - -#define TOML11_CONCATENATE_AUX(x, y) x##y -#define TOML11_CONCATENATE(x, y) TOML11_CONCATENATE_AUX(x, y) - -// ============================================================================ -// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE - -#ifndef TOML11_WITHOUT_DEFINE_NON_INTRUSIVE - -// ---------------------------------------------------------------------------- -// TOML11_ARGS_SIZE - -#define TOML11_INDEX_RSEQ() \ - 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, \ - 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 -#define TOML11_ARGS_SIZE_IMPL(\ - ARG1, ARG2, ARG3, ARG4, ARG5, ARG6, ARG7, ARG8, ARG9, ARG10, \ - ARG11, ARG12, ARG13, ARG14, ARG15, ARG16, ARG17, ARG18, ARG19, ARG20, \ - ARG21, ARG22, ARG23, ARG24, ARG25, ARG26, ARG27, ARG28, ARG29, ARG30, \ - ARG31, ARG32, N, ...) N -#define TOML11_ARGS_SIZE_AUX(...) TOML11_ARGS_SIZE_IMPL(__VA_ARGS__) -#define TOML11_ARGS_SIZE(...) TOML11_ARGS_SIZE_AUX(__VA_ARGS__, TOML11_INDEX_RSEQ()) - -// ---------------------------------------------------------------------------- -// TOML11_FOR_EACH_VA_ARGS - -#define TOML11_FOR_EACH_VA_ARGS_AUX_1( FUNCTOR, ARG1 ) FUNCTOR(ARG1) -#define TOML11_FOR_EACH_VA_ARGS_AUX_2( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_1( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_3( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_2( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_4( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_3( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_5( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_4( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_6( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_5( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_7( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_6( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_8( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_7( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_9( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_8( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_10(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_9( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_11(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_10(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_12(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_11(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_13(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_12(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_14(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_13(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_15(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_14(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_16(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_15(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_17(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_16(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_18(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_17(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_19(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_18(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_20(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_19(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_21(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_20(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_22(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_21(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_23(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_22(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_24(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_23(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_25(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_24(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_26(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_25(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_27(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_26(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_28(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_27(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_29(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_28(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_30(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_29(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_31(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_30(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_32(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_31(FUNCTOR, __VA_ARGS__) - -#define TOML11_FOR_EACH_VA_ARGS(FUNCTOR, ...)\ - TOML11_CONCATENATE(TOML11_FOR_EACH_VA_ARGS_AUX_, TOML11_ARGS_SIZE(__VA_ARGS__))(FUNCTOR, __VA_ARGS__) - -// ---------------------------------------------------------------------------- -// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE - -// use it in the following way. -// ```cpp -// namespace foo -// { -// struct Foo -// { -// std::string s; -// double d; -// int i; -// }; -// } // foo -// -// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(foo::Foo, s, d, i) -// ``` -// And then you can use `toml::find(file, "foo");` -// -#define TOML11_FIND_MEMBER_VARIABLE_FROM_VALUE(VAR_NAME)\ - obj.VAR_NAME = toml::find(v, TOML11_STRINGIZE(VAR_NAME)); - -#define TOML11_ASSIGN_MEMBER_VARIABLE_TO_VALUE(VAR_NAME)\ - v[TOML11_STRINGIZE(VAR_NAME)] = obj.VAR_NAME; - -#define TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(NAME, ...)\ - namespace toml { \ - template<> \ - struct from \ - { \ - template class T, \ - template class A> \ - static NAME from_toml(const basic_value& v) \ - { \ - NAME obj; \ - TOML11_FOR_EACH_VA_ARGS(TOML11_FIND_MEMBER_VARIABLE_FROM_VALUE, __VA_ARGS__) \ - return obj; \ - } \ - }; \ - template<> \ - struct into \ - { \ - static value into_toml(const NAME& obj) \ - { \ - ::toml::value v = ::toml::table{}; \ - TOML11_FOR_EACH_VA_ARGS(TOML11_ASSIGN_MEMBER_VARIABLE_TO_VALUE, __VA_ARGS__) \ - return v; \ - } \ - }; \ - } /* toml */ - -#endif// TOML11_WITHOUT_DEFINE_NON_INTRUSIVE - -#endif// TOML11_MACROS_HPP diff -Nru nix-2.8.0/src/toml11/toml/parser.hpp nix-2.16.1+dfsg/src/toml11/toml/parser.hpp --- nix-2.8.0/src/toml11/toml/parser.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/parser.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,2364 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_PARSER_HPP -#define TOML11_PARSER_HPP -#include -#include -#include - -#include "combinator.hpp" -#include "lexer.hpp" -#include "region.hpp" -#include "result.hpp" -#include "types.hpp" -#include "value.hpp" - -#ifndef TOML11_DISABLE_STD_FILESYSTEM -#ifdef __cpp_lib_filesystem -#if __has_include() -#define TOML11_HAS_STD_FILESYSTEM -#include -#endif // has_include() -#endif // __cpp_lib_filesystem -#endif // TOML11_DISABLE_STD_FILESYSTEM - -namespace toml -{ -namespace detail -{ - -inline result, std::string> -parse_boolean(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_boolean::invoke(loc)) - { - const auto reg = token.unwrap(); - if (reg.str() == "true") {return ok(std::make_pair(true, reg));} - else if(reg.str() == "false") {return ok(std::make_pair(false, reg));} - else // internal error. - { - throw internal_error(format_underline( - "toml::parse_boolean: internal error", - {{source_location(reg), "invalid token"}}), - source_location(reg)); - } - } - loc.reset(first); //rollback - return err(format_underline("toml::parse_boolean: ", - {{source_location(loc), "the next token is not a boolean"}})); -} - -inline result, std::string> -parse_binary_integer(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_bin_int::invoke(loc)) - { - auto str = token.unwrap().str(); - assert(str.size() > 2); // minimum -> 0b1 - integer retval(0), base(1); - for(auto i(str.rbegin()), e(str.rend() - 2); i!=e; ++i) - { - if (*i == '1'){retval += base; base *= 2;} - else if(*i == '0'){base *= 2;} - else if(*i == '_'){/* do nothing. */} - else // internal error. - { - throw internal_error(format_underline( - "toml::parse_integer: internal error", - {{source_location(token.unwrap()), "invalid token"}}), - source_location(loc)); - } - } - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_binary_integer:", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_octal_integer(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_oct_int::invoke(loc)) - { - auto str = token.unwrap().str(); - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - str.erase(str.begin()); str.erase(str.begin()); // remove `0o` prefix - - std::istringstream iss(str); - integer retval(0); - iss >> std::oct >> retval; - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_octal_integer:", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_hexadecimal_integer(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_hex_int::invoke(loc)) - { - auto str = token.unwrap().str(); - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - str.erase(str.begin()); str.erase(str.begin()); // remove `0x` prefix - - std::istringstream iss(str); - integer retval(0); - iss >> std::hex >> retval; - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_hexadecimal_integer", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_integer(location& loc) -{ - const auto first = loc.iter(); - if(first != loc.end() && *first == '0') - { - const auto second = std::next(first); - if(second == loc.end()) // the token is just zero. - { - loc.advance(); - return ok(std::make_pair(0, region(loc, first, second))); - } - - if(*second == 'b') {return parse_binary_integer (loc);} // 0b1100 - if(*second == 'o') {return parse_octal_integer (loc);} // 0o775 - if(*second == 'x') {return parse_hexadecimal_integer(loc);} // 0xC0FFEE - - if(std::isdigit(*second)) - { - return err(format_underline("toml::parse_integer: " - "leading zero in an Integer is not allowed.", - {{source_location(loc), "leading zero"}})); - } - else if(std::isalpha(*second)) - { - return err(format_underline("toml::parse_integer: " - "unknown integer prefix appeared.", - {{source_location(loc), "none of 0x, 0o, 0b"}})); - } - } - - if(const auto token = lex_dec_int::invoke(loc)) - { - auto str = token.unwrap().str(); - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - - std::istringstream iss(str); - integer retval(0); - iss >> retval; - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_integer: ", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_floating(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_float::invoke(loc)) - { - auto str = token.unwrap().str(); - if(str == "inf" || str == "+inf") - { - if(std::numeric_limits::has_infinity) - { - return ok(std::make_pair( - std::numeric_limits::infinity(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: inf value found" - " but the current environment does not support inf. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - else if(str == "-inf") - { - if(std::numeric_limits::has_infinity) - { - return ok(std::make_pair( - -std::numeric_limits::infinity(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: inf value found" - " but the current environment does not support inf. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - else if(str == "nan" || str == "+nan") - { - if(std::numeric_limits::has_quiet_NaN) - { - return ok(std::make_pair( - std::numeric_limits::quiet_NaN(), token.unwrap())); - } - else if(std::numeric_limits::has_signaling_NaN) - { - return ok(std::make_pair( - std::numeric_limits::signaling_NaN(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: NaN value found" - " but the current environment does not support NaN. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - else if(str == "-nan") - { - if(std::numeric_limits::has_quiet_NaN) - { - return ok(std::make_pair( - -std::numeric_limits::quiet_NaN(), token.unwrap())); - } - else if(std::numeric_limits::has_signaling_NaN) - { - return ok(std::make_pair( - -std::numeric_limits::signaling_NaN(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: NaN value found" - " but the current environment does not support NaN. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - std::istringstream iss(str); - floating v(0.0); - iss >> v; - return ok(std::make_pair(v, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_floating: ", - {{source_location(loc), "the next token is not a float"}})); -} - -inline std::string read_utf8_codepoint(const region& reg, const location& loc) -{ - const auto str = reg.str().substr(1); - std::uint_least32_t codepoint; - std::istringstream iss(str); - iss >> std::hex >> codepoint; - - const auto to_char = [](const std::uint_least32_t i) noexcept -> char { - const auto uc = static_cast(i); - return *reinterpret_cast(std::addressof(uc)); - }; - - std::string character; - if(codepoint < 0x80) // U+0000 ... U+0079 ; just an ASCII. - { - character += static_cast(codepoint); - } - else if(codepoint < 0x800) //U+0080 ... U+07FF - { - // 110yyyyx 10xxxxxx; 0x3f == 0b0011'1111 - character += to_char(0xC0| codepoint >> 6); - character += to_char(0x80|(codepoint & 0x3F)); - } - else if(codepoint < 0x10000) // U+0800...U+FFFF - { - if(0xD800 <= codepoint && codepoint <= 0xDFFF) - { - throw syntax_error(format_underline( - "toml::read_utf8_codepoint: codepoints in the range " - "[0xD800, 0xDFFF] are not valid UTF-8.", {{ - source_location(loc), "not a valid UTF-8 codepoint" - }}), source_location(loc)); - } - assert(codepoint < 0xD800 || 0xDFFF < codepoint); - // 1110yyyy 10yxxxxx 10xxxxxx - character += to_char(0xE0| codepoint >> 12); - character += to_char(0x80|(codepoint >> 6 & 0x3F)); - character += to_char(0x80|(codepoint & 0x3F)); - } - else if(codepoint < 0x110000) // U+010000 ... U+10FFFF - { - // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx - character += to_char(0xF0| codepoint >> 18); - character += to_char(0x80|(codepoint >> 12 & 0x3F)); - character += to_char(0x80|(codepoint >> 6 & 0x3F)); - character += to_char(0x80|(codepoint & 0x3F)); - } - else // out of UTF-8 region - { - throw syntax_error(format_underline("toml::read_utf8_codepoint:" - " input codepoint is too large.", - {{source_location(loc), "should be in [0x00..0x10FFFF]"}}), - source_location(loc)); - } - return character; -} - -inline result parse_escape_sequence(location& loc) -{ - const auto first = loc.iter(); - if(first == loc.end() || *first != '\\') - { - return err(format_underline("toml::parse_escape_sequence: ", {{ - source_location(loc), "the next token is not a backslash \"\\\""}})); - } - loc.advance(); - switch(*loc.iter()) - { - case '\\':{loc.advance(); return ok(std::string("\\"));} - case '"' :{loc.advance(); return ok(std::string("\""));} - case 'b' :{loc.advance(); return ok(std::string("\b"));} - case 't' :{loc.advance(); return ok(std::string("\t"));} - case 'n' :{loc.advance(); return ok(std::string("\n"));} - case 'f' :{loc.advance(); return ok(std::string("\f"));} - case 'r' :{loc.advance(); return ok(std::string("\r"));} - case 'u' : - { - if(const auto token = lex_escape_unicode_short::invoke(loc)) - { - return ok(read_utf8_codepoint(token.unwrap(), loc)); - } - else - { - return err(format_underline("parse_escape_sequence: " - "invalid token found in UTF-8 codepoint uXXXX.", - {{source_location(loc), "here"}})); - } - } - case 'U': - { - if(const auto token = lex_escape_unicode_long::invoke(loc)) - { - return ok(read_utf8_codepoint(token.unwrap(), loc)); - } - else - { - return err(format_underline("parse_escape_sequence: " - "invalid token found in UTF-8 codepoint Uxxxxxxxx", - {{source_location(loc), "here"}})); - } - } - } - - const auto msg = format_underline("parse_escape_sequence: " - "unknown escape sequence appeared.", {{source_location(loc), - "escape sequence is one of \\, \", b, t, n, f, r, uxxxx, Uxxxxxxxx"}}, - /* Hints = */{"if you want to write backslash as just one backslash, " - "use literal string like: regex = '<\\i\\c*\\s*>'"}); - loc.reset(first); - return err(msg); -} - -inline std::ptrdiff_t check_utf8_validity(const std::string& reg) -{ - location loc("tmp", reg); - const auto u8 = repeat::invoke(loc); - if(!u8 || loc.iter() != loc.end()) - { - const auto error_location = std::distance(loc.begin(), loc.iter()); - assert(0 <= error_location); - return error_location; - } - return -1; -} - -inline result, std::string> -parse_ml_basic_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_ml_basic_string::invoke(loc)) - { - auto inner_loc = loc; - inner_loc.reset(first); - - std::string retval; - retval.reserve(token.unwrap().size()); - - auto delim = lex_ml_basic_string_open::invoke(inner_loc); - if(!delim) - { - throw internal_error(format_underline( - "parse_ml_basic_string: invalid token", - {{source_location(inner_loc), "should be \"\"\""}}), - source_location(inner_loc)); - } - // immediate newline is ignored (if exists) - /* discard return value */ lex_newline::invoke(inner_loc); - - delim = none(); - while(!delim) - { - using lex_unescaped_seq = repeat< - either, unlimited>; - if(auto unescaped = lex_unescaped_seq::invoke(inner_loc)) - { - retval += unescaped.unwrap().str(); - } - if(auto escaped = parse_escape_sequence(inner_loc)) - { - retval += escaped.unwrap(); - } - if(auto esc_nl = lex_ml_basic_escaped_newline::invoke(inner_loc)) - { - // ignore newline after escape until next non-ws char - } - if(inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "parse_ml_basic_string: unexpected end of region", - {{source_location(inner_loc), "not sufficient token"}}), - source_location(inner_loc)); - } - delim = lex_ml_basic_string_close::invoke(inner_loc); - } - // `lex_ml_basic_string_close` allows 3 to 5 `"`s to allow 1 or 2 `"`s - // at just before the delimiter. Here, we need to attach `"`s at the - // end of the string body, if it exists. - // For detail, see the definition of `lex_ml_basic_string_close`. - assert(std::all_of(delim.unwrap().first(), delim.unwrap().last(), - [](const char c) noexcept {return c == '\"';})); - switch(delim.unwrap().size()) - { - case 3: {break;} - case 4: {retval += "\""; break;} - case 5: {retval += "\"\""; break;} - default: - { - throw internal_error(format_underline( - "parse_ml_basic_string: closing delimiter has invalid length", - {{source_location(inner_loc), "end of this"}}), - source_location(inner_loc)); - } - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair(toml::string(retval), token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_ml_basic_string: " - "the next token is not a valid multiline string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_basic_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_basic_string::invoke(loc)) - { - auto inner_loc = loc; - inner_loc.reset(first); - - auto quot = lex_quotation_mark::invoke(inner_loc); - if(!quot) - { - throw internal_error(format_underline("parse_basic_string: " - "invalid token", {{source_location(inner_loc), "should be \""}}), - source_location(inner_loc)); - } - - std::string retval; - retval.reserve(token.unwrap().size()); - - quot = none(); - while(!quot) - { - using lex_unescaped_seq = repeat; - if(auto unescaped = lex_unescaped_seq::invoke(inner_loc)) - { - retval += unescaped.unwrap().str(); - } - if(auto escaped = parse_escape_sequence(inner_loc)) - { - retval += escaped.unwrap(); - } - if(inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "parse_basic_string: unexpected end of region", - {{source_location(inner_loc), "not sufficient token"}}), - source_location(inner_loc)); - } - quot = lex_quotation_mark::invoke(inner_loc); - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair(toml::string(retval), token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); // rollback - return err(format_underline("toml::parse_basic_string: " - "the next token is not a valid string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_ml_literal_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_ml_literal_string::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_ml_literal_string_open::invoke(inner_loc); - if(!open) - { - throw internal_error(format_underline( - "parse_ml_literal_string: invalid token", - {{source_location(inner_loc), "should be '''"}}), - source_location(inner_loc)); - } - // immediate newline is ignored (if exists) - /* discard return value */ lex_newline::invoke(inner_loc); - - const auto body = lex_ml_literal_body::invoke(inner_loc); - - const auto close = lex_ml_literal_string_close::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "parse_ml_literal_string: invalid token", - {{source_location(inner_loc), "should be '''"}}), - source_location(inner_loc)); - } - // `lex_ml_literal_string_close` allows 3 to 5 `'`s to allow 1 or 2 `'`s - // at just before the delimiter. Here, we need to attach `'`s at the - // end of the string body, if it exists. - // For detail, see the definition of `lex_ml_basic_string_close`. - - std::string retval = body.unwrap().str(); - assert(std::all_of(close.unwrap().first(), close.unwrap().last(), - [](const char c) noexcept {return c == '\'';})); - switch(close.unwrap().size()) - { - case 3: {break;} - case 4: {retval += "'"; break;} - case 5: {retval += "''"; break;} - default: - { - throw internal_error(format_underline( - "parse_ml_literal_string: closing delimiter has invalid length", - {{source_location(inner_loc), "end of this"}}), - source_location(inner_loc)); - } - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair(toml::string(retval, toml::string_t::literal), - token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); // rollback - return err(format_underline("toml::parse_ml_literal_string: " - "the next token is not a valid multiline literal string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_literal_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_literal_string::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_apostrophe::invoke(inner_loc); - if(!open) - { - throw internal_error(format_underline( - "parse_literal_string: invalid token", - {{source_location(inner_loc), "should be '"}}), - source_location(inner_loc)); - } - - const auto body = repeat::invoke(inner_loc); - - const auto close = lex_apostrophe::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "parse_literal_string: invalid token", - {{source_location(inner_loc), "should be '"}}), - source_location(inner_loc)); - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair( - toml::string(body.unwrap().str(), toml::string_t::literal), - token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); // rollback - return err(format_underline("toml::parse_literal_string: " - "the next token is not a valid literal string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_string(location& loc) -{ - if(loc.iter() != loc.end() && *(loc.iter()) == '"') - { - if(loc.iter() + 1 != loc.end() && *(loc.iter() + 1) == '"' && - loc.iter() + 2 != loc.end() && *(loc.iter() + 2) == '"') - { - return parse_ml_basic_string(loc); - } - else - { - return parse_basic_string(loc); - } - } - else if(loc.iter() != loc.end() && *(loc.iter()) == '\'') - { - if(loc.iter() + 1 != loc.end() && *(loc.iter() + 1) == '\'' && - loc.iter() + 2 != loc.end() && *(loc.iter() + 2) == '\'') - { - return parse_ml_literal_string(loc); - } - else - { - return parse_literal_string(loc); - } - } - return err(format_underline("toml::parse_string: ", - {{source_location(loc), "the next token is not a string"}})); -} - -inline result, std::string> -parse_local_date(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_local_date::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto y = lex_date_fullyear::invoke(inner_loc); - if(!y || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != '-') - { - throw internal_error(format_underline( - "toml::parse_inner_local_date: invalid year format", - {{source_location(inner_loc), "should be `-`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto m = lex_date_month::invoke(inner_loc); - if(!m || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != '-') - { - throw internal_error(format_underline( - "toml::parse_local_date: invalid month format", - {{source_location(inner_loc), "should be `-`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto d = lex_date_mday::invoke(inner_loc); - if(!d) - { - throw internal_error(format_underline( - "toml::parse_local_date: invalid day format", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - - const auto year = static_cast(from_string(y.unwrap().str(), 0)); - const auto month = static_cast(from_string(m.unwrap().str(), 0)); - const auto day = static_cast(from_string(d.unwrap().str(), 0)); - - // We briefly check whether the input date is valid or not. But here, we - // only check if the RFC3339 compliance. - // Actually there are several special date that does not exist, - // because of historical reasons, such as 1582/10/5-1582/10/14 (only in - // several countries). But here, we do not care about such a complicated - // rule. It makes the code complicated and there is only low probability - // that such a specific date is needed in practice. If someone need to - // validate date accurately, that means that the one need a specialized - // library for their purpose in a different layer. - { - const bool is_leap = (year % 4 == 0) && ((year % 100 != 0) || (year % 400 == 0)); - const auto max_day = (month == 2) ? (is_leap ? 29 : 28) : - ((month == 4 || month == 6 || month == 9 || month == 11) ? 30 : 31); - - if((month < 1 || 12 < month) || (day < 1 || max_day < day)) - { - throw syntax_error(format_underline("toml::parse_date: " - "invalid date: it does not conform RFC3339.", {{ - source_location(loc), "month should be 01-12, day should be" - " 01-28,29,30,31, depending on month/year." - }}), source_location(inner_loc)); - } - } - return ok(std::make_pair(local_date(year, static_cast(month - 1), day), - token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_local_date: ", - {{source_location(loc), "the next token is not a local_date"}})); - } -} - -inline result, std::string> -parse_local_time(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_local_time::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto h = lex_time_hour::invoke(inner_loc); - if(!h || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != ':') - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid year format", - {{source_location(inner_loc), "should be `:`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto m = lex_time_minute::invoke(inner_loc); - if(!m || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != ':') - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid month format", - {{source_location(inner_loc), "should be `:`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto s = lex_time_second::invoke(inner_loc); - if(!s) - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid second format", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - - const int hour = from_string(h.unwrap().str(), 0); - const int minute = from_string(m.unwrap().str(), 0); - const int second = from_string(s.unwrap().str(), 0); - - if((hour < 0 || 23 < hour) || (minute < 0 || 59 < minute) || - (second < 0 || 60 < second)) // it may be leap second - { - throw syntax_error(format_underline("toml::parse_time: " - "invalid time: it does not conform RFC3339.", {{ - source_location(loc), "hour should be 00-23, minute should be" - " 00-59, second should be 00-60 (depending on the leap" - " second rules.)"}}), source_location(inner_loc)); - } - - local_time time(hour, minute, second, 0, 0); - - const auto before_secfrac = inner_loc.iter(); - if(const auto secfrac = lex_time_secfrac::invoke(inner_loc)) - { - auto sf = secfrac.unwrap().str(); - sf.erase(sf.begin()); // sf.front() == '.' - switch(sf.size() % 3) - { - case 2: sf += '0'; break; - case 1: sf += "00"; break; - case 0: break; - default: break; - } - if(sf.size() >= 9) - { - time.millisecond = from_string(sf.substr(0, 3), 0u); - time.microsecond = from_string(sf.substr(3, 3), 0u); - time.nanosecond = from_string(sf.substr(6, 3), 0u); - } - else if(sf.size() >= 6) - { - time.millisecond = from_string(sf.substr(0, 3), 0u); - time.microsecond = from_string(sf.substr(3, 3), 0u); - } - else if(sf.size() >= 3) - { - time.millisecond = from_string(sf, 0u); - time.microsecond = 0u; - } - } - else - { - if(before_secfrac != inner_loc.iter()) - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid subsecond format", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - return ok(std::make_pair(time, token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_local_time: ", - {{source_location(loc), "the next token is not a local_time"}})); - } -} - -inline result, std::string> -parse_local_datetime(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_local_date_time::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - const auto date = parse_local_date(inner_loc); - if(!date || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_local_datetime: invalid datetime format", - {{source_location(inner_loc), "date, not datetime"}}), - source_location(inner_loc)); - } - const char delim = *(inner_loc.iter()); - if(delim != 'T' && delim != 't' && delim != ' ') - { - throw internal_error(format_underline( - "toml::parse_local_datetime: invalid datetime format", - {{source_location(inner_loc), "should be `T` or ` ` (space)"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto time = parse_local_time(inner_loc); - if(!time) - { - throw internal_error(format_underline( - "toml::parse_local_datetime: invalid datetime format", - {{source_location(inner_loc), "invalid time format"}}), - source_location(inner_loc)); - } - return ok(std::make_pair( - local_datetime(date.unwrap().first, time.unwrap().first), - token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_local_datetime: ", - {{source_location(loc), "the next token is not a local_datetime"}})); - } -} - -inline result, std::string> -parse_offset_datetime(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_offset_date_time::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - const auto datetime = parse_local_datetime(inner_loc); - if(!datetime || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_offset_datetime: invalid datetime format", - {{source_location(inner_loc), "date, not datetime"}}), - source_location(inner_loc)); - } - time_offset offset(0, 0); - if(const auto ofs = lex_time_numoffset::invoke(inner_loc)) - { - const auto str = ofs.unwrap().str(); - - const auto hour = from_string(str.substr(1,2), 0); - const auto minute = from_string(str.substr(4,2), 0); - - if((hour < 0 || 23 < hour) || (minute < 0 || 59 < minute)) - { - throw syntax_error(format_underline("toml::parse_offset_datetime: " - "invalid offset: it does not conform RFC3339.", {{ - source_location(loc), "month should be 01-12, day should be" - " 01-28,29,30,31, depending on month/year." - }}), source_location(inner_loc)); - } - - if(str.front() == '+') - { - offset = time_offset(hour, minute); - } - else - { - offset = time_offset(-hour, -minute); - } - } - else if(*inner_loc.iter() != 'Z' && *inner_loc.iter() != 'z') - { - throw internal_error(format_underline( - "toml::parse_offset_datetime: invalid datetime format", - {{source_location(inner_loc), "should be `Z` or `+HH:MM`"}}), - source_location(inner_loc)); - } - return ok(std::make_pair(offset_datetime(datetime.unwrap().first, offset), - token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_offset_datetime: ", - {{source_location(loc), "the next token is not a offset_datetime"}})); - } -} - -inline result, std::string> -parse_simple_key(location& loc) -{ - if(const auto bstr = parse_basic_string(loc)) - { - return ok(std::make_pair(bstr.unwrap().first.str, bstr.unwrap().second)); - } - if(const auto lstr = parse_literal_string(loc)) - { - return ok(std::make_pair(lstr.unwrap().first.str, lstr.unwrap().second)); - } - if(const auto bare = lex_unquoted_key::invoke(loc)) - { - const auto reg = bare.unwrap(); - return ok(std::make_pair(reg.str(), reg)); - } - return err(format_underline("toml::parse_simple_key: ", - {{source_location(loc), "the next token is not a simple key"}})); -} - -// dotted key become vector of keys -inline result, region>, std::string> -parse_key(location& loc) -{ - const auto first = loc.iter(); - // dotted key -> `foo.bar.baz` where several single keys are chained by - // dots. Whitespaces between keys and dots are allowed. - if(const auto token = lex_dotted_key::invoke(loc)) - { - const auto reg = token.unwrap(); - location inner_loc(loc.name(), reg.str()); - std::vector keys; - - while(inner_loc.iter() != inner_loc.end()) - { - lex_ws::invoke(inner_loc); - if(const auto k = parse_simple_key(inner_loc)) - { - keys.push_back(k.unwrap().first); - } - else - { - throw internal_error(format_underline( - "toml::detail::parse_key: dotted key contains invalid key", - {{source_location(inner_loc), k.unwrap_err()}}), - source_location(inner_loc)); - } - - lex_ws::invoke(inner_loc); - if(inner_loc.iter() == inner_loc.end()) - { - break; - } - else if(*inner_loc.iter() == '.') - { - inner_loc.advance(); // to skip `.` - } - else - { - throw internal_error(format_underline("toml::parse_key: " - "dotted key contains invalid key ", - {{source_location(inner_loc), "should be `.`"}}), - source_location(inner_loc)); - } - } - return ok(std::make_pair(keys, reg)); - } - loc.reset(first); - - // simple_key: a single (basic_string|literal_string|bare key) - if(const auto smpl = parse_simple_key(loc)) - { - return ok(std::make_pair(std::vector(1, smpl.unwrap().first), - smpl.unwrap().second)); - } - return err(format_underline("toml::parse_key: an invalid key appeared.", - {{source_location(loc), "is not a valid key"}}, { - "bare keys : non-empty strings composed only of [A-Za-z0-9_-].", - "quoted keys: same as \"basic strings\" or 'literal strings'.", - "dotted keys: sequence of bare or quoted keys joined with a dot." - })); -} - -// forward-decl to implement parse_array and parse_table -template -result parse_value(location&); - -template -result, std::string> -parse_array(location& loc) -{ - using value_type = Value; - using array_type = typename value_type::array_type; - - const auto first = loc.iter(); - if(loc.iter() == loc.end()) - { - return err("toml::parse_array: input is empty"); - } - if(*loc.iter() != '[') - { - return err("toml::parse_array: token is not an array"); - } - loc.advance(); - - using lex_ws_comment_newline = repeat< - either, unlimited>; - - array_type retval; - while(loc.iter() != loc.end()) - { - lex_ws_comment_newline::invoke(loc); // skip - - if(loc.iter() != loc.end() && *loc.iter() == ']') - { - loc.advance(); // skip ']' - return ok(std::make_pair(retval, - region(loc, first, loc.iter()))); - } - - if(auto val = parse_value(loc)) - { - // After TOML v1.0.0-rc.1, array becomes to be able to have values - // with different types. So here we will omit this by default. - // - // But some of the test-suite checks if the parser accepts a hetero- - // geneous arrays, so we keep this for a while. -#ifdef TOML11_DISALLOW_HETEROGENEOUS_ARRAYS - if(!retval.empty() && retval.front().type() != val.as_ok().type()) - { - auto array_start_loc = loc; - array_start_loc.reset(first); - - throw syntax_error(format_underline("toml::parse_array: " - "type of elements should be the same each other.", { - {source_location(array_start_loc), "array starts here"}, - { - retval.front().location(), - "value has type " + stringize(retval.front().type()) - }, - { - val.unwrap().location(), - "value has different type, " + stringize(val.unwrap().type()) - } - }), source_location(loc)); - } -#endif - retval.push_back(std::move(val.unwrap())); - } - else - { - auto array_start_loc = loc; - array_start_loc.reset(first); - - throw syntax_error(format_underline("toml::parse_array: " - "value having invalid format appeared in an array", { - {source_location(array_start_loc), "array starts here"}, - {source_location(loc), "it is not a valid value."} - }), source_location(loc)); - } - - using lex_array_separator = sequence, character<','>>; - const auto sp = lex_array_separator::invoke(loc); - if(!sp) - { - lex_ws_comment_newline::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == ']') - { - loc.advance(); // skip ']' - return ok(std::make_pair(retval, - region(loc, first, loc.iter()))); - } - else - { - auto array_start_loc = loc; - array_start_loc.reset(first); - - throw syntax_error(format_underline("toml::parse_array:" - " missing array separator `,` after a value", { - {source_location(array_start_loc), "array starts here"}, - {source_location(loc), "should be `,`"} - }), source_location(loc)); - } - } - } - loc.reset(first); - throw syntax_error(format_underline("toml::parse_array: " - "array did not closed by `]`", - {{source_location(loc), "should be closed"}}), - source_location(loc)); -} - -template -result, region>, Value>, std::string> -parse_key_value_pair(location& loc) -{ - using value_type = Value; - - const auto first = loc.iter(); - auto key_reg = parse_key(loc); - if(!key_reg) - { - std::string msg = std::move(key_reg.unwrap_err()); - // if the next token is keyvalue-separator, it means that there are no - // key. then we need to show error as "empty key is not allowed". - if(const auto keyval_sep = lex_keyval_sep::invoke(loc)) - { - loc.reset(first); - msg = format_underline("toml::parse_key_value_pair: " - "empty key is not allowed.", - {{source_location(loc), "key expected before '='"}}); - } - return err(std::move(msg)); - } - - const auto kvsp = lex_keyval_sep::invoke(loc); - if(!kvsp) - { - std::string msg; - // if the line contains '=' after the invalid sequence, possibly the - // error is in the key (like, invalid character in bare key). - const auto line_end = std::find(loc.iter(), loc.end(), '\n'); - if(std::find(loc.iter(), line_end, '=') != line_end) - { - msg = format_underline("toml::parse_key_value_pair: " - "invalid format for key", - {{source_location(loc), "invalid character in key"}}, - {"Did you forget '.' to separate dotted-key?", - "Allowed characters for bare key are [0-9a-zA-Z_-]."}); - } - else // if not, the error is lack of key-value separator. - { - msg = format_underline("toml::parse_key_value_pair: " - "missing key-value separator `=`", - {{source_location(loc), "should be `=`"}}); - } - loc.reset(first); - return err(std::move(msg)); - } - - const auto after_kvsp = loc.iter(); // err msg - auto val = parse_value(loc); - if(!val) - { - std::string msg; - loc.reset(after_kvsp); - // check there is something not a comment/whitespace after `=` - if(sequence, maybe, lex_newline>::invoke(loc)) - { - loc.reset(after_kvsp); - msg = format_underline("toml::parse_key_value_pair: " - "missing value after key-value separator '='", - {{source_location(loc), "expected value, but got nothing"}}); - } - else // there is something not a comment/whitespace, so invalid format. - { - msg = std::move(val.unwrap_err()); - } - loc.reset(first); - return err(msg); - } - return ok(std::make_pair(std::move(key_reg.unwrap()), - std::move(val.unwrap()))); -} - -// for error messages. -template -std::string format_dotted_keys(InputIterator first, const InputIterator last) -{ - static_assert(std::is_same::value_type>::value,""); - - std::string retval(*first++); - for(; first != last; ++first) - { - retval += '.'; - retval += *first; - } - return retval; -} - -// forward decl for is_valid_forward_table_definition -result, region>, std::string> -parse_table_key(location& loc); -template -result, std::string> -parse_inline_table(location& loc); - -// The following toml file is allowed. -// ```toml -// [a.b.c] # here, table `a` has element `b`. -// foo = "bar" -// [a] # merge a = {baz = "qux"} to a = {b = {...}} -// baz = "qux" -// ``` -// But the following is not allowed. -// ```toml -// [a] -// b.c.foo = "bar" -// [a] # error! the same table [a] defined! -// baz = "qux" -// ``` -// The following is neither allowed. -// ```toml -// a = { b.c.foo = "bar"} -// [a] # error! the same table [a] defined! -// baz = "qux" -// ``` -// Here, it parses region of `tab->at(k)` as a table key and check the depth -// of the key. If the key region points deeper node, it would be allowed. -// Otherwise, the key points the same node. It would be rejected. -template -bool is_valid_forward_table_definition(const Value& fwd, const Value& inserting, - Iterator key_first, Iterator key_curr, Iterator key_last) -{ - // ------------------------------------------------------------------------ - // check type of the value to be inserted/merged - - std::string inserting_reg = ""; - if(const auto ptr = detail::get_region(inserting)) - { - inserting_reg = ptr->str(); - } - location inserting_def("internal", std::move(inserting_reg)); - if(const auto inlinetable = parse_inline_table(inserting_def)) - { - // check if we are overwriting existing table. - // ```toml - // # NG - // a.b = 42 - // a = {d = 3.14} - // ``` - // Inserting an inline table to a existing super-table is not allowed in - // any case. If we found it, we can reject it without further checking. - return false; - } - - // ------------------------------------------------------------------------ - // check table defined before - - std::string internal = ""; - if(const auto ptr = detail::get_region(fwd)) - { - internal = ptr->str(); - } - location def("internal", std::move(internal)); - if(const auto tabkeys = parse_table_key(def)) // [table.key] - { - // table keys always contains all the nodes from the root. - const auto& tks = tabkeys.unwrap().first; - if(std::size_t(std::distance(key_first, key_last)) == tks.size() && - std::equal(tks.begin(), tks.end(), key_first)) - { - // the keys are equivalent. it is not allowed. - return false; - } - // the keys are not equivalent. it is allowed. - return true; - } - if(const auto dotkeys = parse_key(def)) - { - // consider the following case. - // [a] - // b.c = {d = 42} - // [a.b.c] - // e = 2.71 - // this defines the table [a.b.c] twice. no? - - // a dotted key starts from the node representing a table in which the - // dotted key belongs to. - const auto& dks = dotkeys.unwrap().first; - if(std::size_t(std::distance(key_curr, key_last)) == dks.size() && - std::equal(dks.begin(), dks.end(), key_curr)) - { - // the keys are equivalent. it is not allowed. - return false; - } - // the keys are not equivalent. it is allowed. - return true; - } - return false; -} - -template -result -insert_nested_key(typename Value::table_type& root, const Value& v, - InputIterator iter, const InputIterator last, - region key_reg, - const bool is_array_of_table = false) -{ - static_assert(std::is_same::value_type>::value,""); - - using value_type = Value; - using table_type = typename value_type::table_type; - using array_type = typename value_type::array_type; - - const auto first = iter; - assert(iter != last); - - table_type* tab = std::addressof(root); - for(; iter != last; ++iter) // search recursively - { - const key& k = *iter; - if(std::next(iter) == last) // k is the last key - { - // XXX if the value is array-of-tables, there can be several - // tables that are in the same array. in that case, we need to - // find the last element and insert it to there. - if(is_array_of_table) - { - if(tab->count(k) == 1) // there is already an array of table - { - if(tab->at(k).is_table()) - { - // show special err msg for conflicting table - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), - "\") cannot be defined"), { - {tab->at(k).location(), "table already defined"}, - {v.location(), "this conflicts with the previous table"} - }), v.location()); - } - else if(!(tab->at(k).is_array())) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), "\") collides with" - " existing value"), { - {tab->at(k).location(), - concat_to_string("this ", tab->at(k).type(), - " value already exists")}, - {v.location(), - "while inserting this array-of-tables"} - }), v.location()); - } - // the above if-else-if checks tab->at(k) is an array - auto& a = tab->at(k).as_array(); - // If table element is defined as [[array_of_tables]], it - // cannot be an empty array. If an array of tables is - // defined as `aot = []`, it cannot be appended. - if(a.empty() || !(a.front().is_table())) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), "\") collides with" - " existing value"), { - {tab->at(k).location(), - concat_to_string("this ", tab->at(k).type(), - " value already exists")}, - {v.location(), - "while inserting this array-of-tables"} - }), v.location()); - } - // avoid conflicting array of table like the following. - // ```toml - // a = [{b = 42}] # define a as an array of *inline* tables - // [[a]] # a is an array of *multi-line* tables - // b = 54 - // ``` - // Here, from the type information, these cannot be detected - // because inline table is also a table. - // But toml v0.5.0 explicitly says it is invalid. The above - // array-of-tables has a static size and appending to the - // array is invalid. - // In this library, multi-line table value has a region - // that points to the key of the table (e.g. [[a]]). By - // comparing the first two letters in key, we can detect - // the array-of-table is inline or multiline. - if(const auto ptr = detail::get_region(a.front())) - { - if(ptr->str().substr(0,2) != "[[") - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), "\") collides " - "with existing array-of-tables"), { - {tab->at(k).location(), - concat_to_string("this ", tab->at(k).type(), - " value has static size")}, - {v.location(), - "appending it to the statically sized array"} - }), v.location()); - } - } - a.push_back(v); - return ok(true); - } - else // if not, we need to create the array of table - { - // XXX: Consider the following array of tables. - // ```toml - // # This is a comment. - // [[aot]] - // foo = "bar" - // ``` - // Here, the comment is for `aot`. But here, actually two - // values are defined. An array that contains tables, named - // `aot`, and the 0th element of the `aot`, `{foo = "bar"}`. - // Those two are different from each other. But both of them - // points to the same portion of the TOML file, `[[aot]]`, - // so `key_reg.comments()` returns `# This is a comment`. - // If it is assigned as a comment of `aot` defined here, the - // comment will be duplicated. Both the `aot` itself and - // the 0-th element will have the same comment. This causes - // "duplication of the same comments" bug when the data is - // serialized. - // Next, consider the following. - // ```toml - // # comment 1 - // aot = [ - // # comment 2 - // {foo = "bar"}, - // ] - // ``` - // In this case, we can distinguish those two comments. So - // here we need to add "comment 1" to the `aot` and - // "comment 2" to the 0th element of that. - // To distinguish those two, we check the key region. - std::vector comments{/* empty by default */}; - if(key_reg.str().substr(0, 2) != "[[") - { - comments = key_reg.comments(); - } - value_type aot(array_type(1, v), key_reg, std::move(comments)); - tab->insert(std::make_pair(k, aot)); - return ok(true); - } - } // end if(array of table) - - if(tab->count(k) == 1) - { - if(tab->at(k).is_table() && v.is_table()) - { - if(!is_valid_forward_table_definition( - tab->at(k), v, first, iter, last)) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: table (\"", - format_dotted_keys(first, last), - "\") already exists."), { - {tab->at(k).location(), "table already exists here"}, - {v.location(), "table defined twice"} - }), v.location()); - } - // to allow the following toml file. - // [a.b.c] - // d = 42 - // [a] - // e = 2.71 - auto& t = tab->at(k).as_table(); - for(const auto& kv : v.as_table()) - { - if(tab->at(k).contains(kv.first)) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: value (\"", - format_dotted_keys(first, last), - "\") already exists."), { - {t.at(kv.first).location(), "already exists here"}, - {v.location(), "this defined twice"} - }), v.location()); - } - t[kv.first] = kv.second; - } - detail::change_region(tab->at(k), key_reg); - return ok(true); - } - else if(v.is_table() && - tab->at(k).is_array() && - tab->at(k).as_array().size() > 0 && - tab->at(k).as_array().front().is_table()) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of tables (\"", - format_dotted_keys(first, last), "\") already exists."), { - {tab->at(k).location(), "array of tables defined here"}, - {v.location(), "table conflicts with the previous array of table"} - }), v.location()); - } - else - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: value (\"", - format_dotted_keys(first, last), "\") already exists."), { - {tab->at(k).location(), "value already exists here"}, - {v.location(), "value defined twice"} - }), v.location()); - } - } - tab->insert(std::make_pair(k, v)); - return ok(true); - } - else // k is not the last one, we should insert recursively - { - // if there is no corresponding value, insert it first. - // related: you don't need to write - // # [x] - // # [x.y] - // to write - // [x.y.z] - if(tab->count(k) == 0) - { - // a table that is defined implicitly doesn't have any comments. - (*tab)[k] = value_type(table_type{}, key_reg, {/*no comment*/}); - } - - // type checking... - if(tab->at(k).is_table()) - { - // According to toml-lang/toml:36d3091b3 "Clarify that inline - // tables are immutable", check if it adds key-value pair to an - // inline table. - if(const auto* ptr = get_region(tab->at(k))) - { - // here, if the value is a (multi-line) table, the region - // should be something like `[table-name]`. - if(ptr->front() == '{') - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: inserting to an inline table (", - format_dotted_keys(first, std::next(iter)), - ") but inline tables are immutable"), { - {tab->at(k).location(), "inline tables are immutable"}, - {v.location(), "inserting this"} - }), v.location()); - } - } - tab = std::addressof((*tab)[k].as_table()); - } - else if(tab->at(k).is_array()) // inserting to array-of-tables? - { - auto& a = (*tab)[k].as_array(); - if(!a.back().is_table()) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: target (", - format_dotted_keys(first, std::next(iter)), - ") is neither table nor an array of tables"), { - {a.back().location(), concat_to_string( - "actual type is ", a.back().type())}, - {v.location(), "inserting this"} - }), v.location()); - } - tab = std::addressof(a.back().as_table()); - } - else - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: target (", - format_dotted_keys(first, std::next(iter)), - ") is neither table nor an array of tables"), { - {tab->at(k).location(), concat_to_string( - "actual type is ", tab->at(k).type())}, - {v.location(), "inserting this"} - }), v.location()); - } - } - } - return err(std::string("toml::detail::insert_nested_key: never reach here")); -} - -template -result, std::string> -parse_inline_table(location& loc) -{ - using value_type = Value; - using table_type = typename value_type::table_type; - - const auto first = loc.iter(); - table_type retval; - if(!(loc.iter() != loc.end() && *loc.iter() == '{')) - { - return err(format_underline("toml::parse_inline_table: ", - {{source_location(loc), "the next token is not an inline table"}})); - } - loc.advance(); - - // check if the inline table is an empty table = { } - maybe::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == '}') - { - loc.advance(); // skip `}` - return ok(std::make_pair(retval, region(loc, first, loc.iter()))); - } - - // it starts from "{". it should be formatted as inline-table - while(loc.iter() != loc.end()) - { - const auto kv_r = parse_key_value_pair(loc); - if(!kv_r) - { - return err(kv_r.unwrap_err()); - } - - const auto& kvpair = kv_r.unwrap(); - const std::vector& keys = kvpair.first.first; - const auto& key_reg = kvpair.first.second; - const value_type& val = kvpair.second; - - const auto inserted = - insert_nested_key(retval, val, keys.begin(), keys.end(), key_reg); - if(!inserted) - { - throw internal_error("toml::parse_inline_table: " - "failed to insert value into table: " + inserted.unwrap_err(), - source_location(loc)); - } - - using lex_table_separator = sequence, character<','>>; - const auto sp = lex_table_separator::invoke(loc); - - if(!sp) - { - maybe::invoke(loc); - - if(loc.iter() == loc.end()) - { - throw syntax_error(format_underline( - "toml::parse_inline_table: missing table separator `}` ", - {{source_location(loc), "should be `}`"}}), - source_location(loc)); - } - else if(*loc.iter() == '}') - { - loc.advance(); // skip `}` - return ok(std::make_pair( - retval, region(loc, first, loc.iter()))); - } - else if(*loc.iter() == '#' || *loc.iter() == '\r' || *loc.iter() == '\n') - { - throw syntax_error(format_underline( - "toml::parse_inline_table: missing curly brace `}`", - {{source_location(loc), "should be `}`"}}), - source_location(loc)); - } - else - { - throw syntax_error(format_underline( - "toml::parse_inline_table: missing table separator `,` ", - {{source_location(loc), "should be `,`"}}), - source_location(loc)); - } - } - else // `,` is found - { - maybe::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == '}') - { - throw syntax_error(format_underline( - "toml::parse_inline_table: trailing comma is not allowed in" - " an inline table", - {{source_location(loc), "should be `}`"}}), - source_location(loc)); - } - } - } - loc.reset(first); - throw syntax_error(format_underline("toml::parse_inline_table: " - "inline table did not closed by `}`", - {{source_location(loc), "should be closed"}}), - source_location(loc)); -} - -inline result guess_number_type(const location& l) -{ - // This function tries to find some (common) mistakes by checking characters - // that follows the last character of a value. But it is often difficult - // because some non-newline characters can appear after a value. E.g. - // spaces, tabs, commas (in an array or inline table), closing brackets - // (of an array or inline table), comment-sign (#). Since this function - // does not parse further, those characters are always allowed to be there. - location loc = l; - - if(lex_offset_date_time::invoke(loc)) {return ok(value_t::offset_datetime);} - loc.reset(l.iter()); - - if(lex_local_date_time::invoke(loc)) - { - // bad offset may appear after this. - if(loc.iter() != loc.end() && (*loc.iter() == '+' || *loc.iter() == '-' - || *loc.iter() == 'Z' || *loc.iter() == 'z')) - { - return err(format_underline("bad offset: should be [+-]HH:MM or Z", - {{source_location(loc), "[+-]HH:MM or Z"}}, - {"pass: +09:00, -05:30", "fail: +9:00, -5:30"})); - } - return ok(value_t::local_datetime); - } - loc.reset(l.iter()); - - if(lex_local_date::invoke(loc)) - { - // bad time may appear after this. - // A space is allowed as a delimiter between local time. But there are - // both cases in which a space becomes valid or invalid. - // - invalid: 2019-06-16 7:00:00 - // - valid : 2019-06-16 07:00:00 - if(loc.iter() != loc.end()) - { - const auto c = *loc.iter(); - if(c == 'T' || c == 't') - { - return err(format_underline("bad time: should be HH:MM:SS.subsec", - {{source_location(loc), "HH:MM:SS.subsec"}}, - {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", - "fail: 1979-05-27T7:32:00, 1979-05-27 17:32"})); - } - if('0' <= c && c <= '9') - { - return err(format_underline("bad time: missing T", - {{source_location(loc), "T or space required here"}}, - {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", - "fail: 1979-05-27T7:32:00, 1979-05-27 7:32"})); - } - if(c == ' ' && std::next(loc.iter()) != loc.end() && - ('0' <= *std::next(loc.iter()) && *std::next(loc.iter())<= '9')) - { - loc.advance(); - return err(format_underline("bad time: should be HH:MM:SS.subsec", - {{source_location(loc), "HH:MM:SS.subsec"}}, - {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", - "fail: 1979-05-27T7:32:00, 1979-05-27 7:32"})); - } - } - return ok(value_t::local_date); - } - loc.reset(l.iter()); - - if(lex_local_time::invoke(loc)) {return ok(value_t::local_time);} - loc.reset(l.iter()); - - if(lex_float::invoke(loc)) - { - if(loc.iter() != loc.end() && *loc.iter() == '_') - { - return err(format_underline("bad float: `_` should be surrounded by digits", - {{source_location(loc), "here"}}, - {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", - "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); - } - return ok(value_t::floating); - } - loc.reset(l.iter()); - - if(lex_integer::invoke(loc)) - { - if(loc.iter() != loc.end()) - { - const auto c = *loc.iter(); - if(c == '_') - { - return err(format_underline("bad integer: `_` should be surrounded by digits", - {{source_location(loc), "here"}}, - {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", - "fail: 1__000, 0123"})); - } - if('0' <= c && c <= '9') - { - // leading zero. point '0' - loc.retrace(); - return err(format_underline("bad integer: leading zero", - {{source_location(loc), "here"}}, - {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", - "fail: 1__000, 0123"})); - } - if(c == ':' || c == '-') - { - return err(format_underline("bad datetime: invalid format", - {{source_location(loc), "here"}}, - {"pass: 1979-05-27T07:32:00-07:00, 1979-05-27 07:32:00.999999Z", - "fail: 1979-05-27T7:32:00-7:00, 1979-05-27 7:32-00:30"})); - } - if(c == '.' || c == 'e' || c == 'E') - { - return err(format_underline("bad float: invalid format", - {{source_location(loc), "here"}}, - {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", - "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); - } - } - return ok(value_t::integer); - } - if(loc.iter() != loc.end() && *loc.iter() == '.') - { - return err(format_underline("bad float: invalid format", - {{source_location(loc), "integer part required before this"}}, - {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", - "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); - } - if(loc.iter() != loc.end() && *loc.iter() == '_') - { - return err(format_underline("bad number: `_` should be surrounded by digits", - {{source_location(loc), "`_` is not surrounded by digits"}}, - {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", - "fail: 1__000, 0123"})); - } - return err(format_underline("bad format: unknown value appeared", - {{source_location(loc), "here"}})); -} - -inline result guess_value_type(const location& loc) -{ - switch(*loc.iter()) - { - case '"' : {return ok(value_t::string); } - case '\'': {return ok(value_t::string); } - case 't' : {return ok(value_t::boolean); } - case 'f' : {return ok(value_t::boolean); } - case '[' : {return ok(value_t::array); } - case '{' : {return ok(value_t::table); } - case 'i' : {return ok(value_t::floating);} // inf. - case 'n' : {return ok(value_t::floating);} // nan. - default : {return guess_number_type(loc);} - } -} - -template -result -parse_value_helper(result, std::string> rslt) -{ - if(rslt.is_ok()) - { - auto comments = rslt.as_ok().second.comments(); - return ok(Value(std::move(rslt.as_ok()), std::move(comments))); - } - else - { - return err(std::move(rslt.as_err())); - } -} - -template -result parse_value(location& loc) -{ - const auto first = loc.iter(); - if(first == loc.end()) - { - return err(format_underline("toml::parse_value: input is empty", - {{source_location(loc), ""}})); - } - - const auto type = guess_value_type(loc); - if(!type) - { - return err(type.unwrap_err()); - } - - switch(type.unwrap()) - { - case value_t::boolean : {return parse_value_helper(parse_boolean(loc) );} - case value_t::integer : {return parse_value_helper(parse_integer(loc) );} - case value_t::floating : {return parse_value_helper(parse_floating(loc) );} - case value_t::string : {return parse_value_helper(parse_string(loc) );} - case value_t::offset_datetime: {return parse_value_helper(parse_offset_datetime(loc) );} - case value_t::local_datetime : {return parse_value_helper(parse_local_datetime(loc) );} - case value_t::local_date : {return parse_value_helper(parse_local_date(loc) );} - case value_t::local_time : {return parse_value_helper(parse_local_time(loc) );} - case value_t::array : {return parse_value_helper(parse_array(loc) );} - case value_t::table : {return parse_value_helper(parse_inline_table(loc));} - default: - { - const auto msg = format_underline("toml::parse_value: " - "unknown token appeared", {{source_location(loc), "unknown"}}); - loc.reset(first); - return err(msg); - } - } -} - -inline result, region>, std::string> -parse_table_key(location& loc) -{ - if(auto token = lex_std_table::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_std_table_open::invoke(inner_loc); - if(!open || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_table_key: no `[`", - {{source_location(inner_loc), "should be `[`"}}), - source_location(inner_loc)); - } - // to skip [ a . b . c ] - // ^----------- this whitespace - lex_ws::invoke(inner_loc); - const auto keys = parse_key(inner_loc); - if(!keys) - { - throw internal_error(format_underline( - "toml::parse_table_key: invalid key", - {{source_location(inner_loc), "not key"}}), - source_location(inner_loc)); - } - // to skip [ a . b . c ] - // ^-- this whitespace - lex_ws::invoke(inner_loc); - const auto close = lex_std_table_close::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "toml::parse_table_key: no `]`", - {{source_location(inner_loc), "should be `]`"}}), - source_location(inner_loc)); - } - - // after [table.key], newline or EOF(empty table) required. - if(loc.iter() != loc.end()) - { - using lex_newline_after_table_key = - sequence, maybe, lex_newline>; - const auto nl = lex_newline_after_table_key::invoke(loc); - if(!nl) - { - throw syntax_error(format_underline( - "toml::parse_table_key: newline required after [table.key]", - {{source_location(loc), "expected newline"}}), - source_location(loc)); - } - } - return ok(std::make_pair(keys.unwrap().first, token.unwrap())); - } - else - { - return err(format_underline("toml::parse_table_key: " - "not a valid table key", {{source_location(loc), "here"}})); - } -} - -inline result, region>, std::string> -parse_array_table_key(location& loc) -{ - if(auto token = lex_array_table::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_array_table_open::invoke(inner_loc); - if(!open || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_array_table_key: no `[[`", - {{source_location(inner_loc), "should be `[[`"}}), - source_location(inner_loc)); - } - lex_ws::invoke(inner_loc); - const auto keys = parse_key(inner_loc); - if(!keys) - { - throw internal_error(format_underline( - "toml::parse_array_table_key: invalid key", - {{source_location(inner_loc), "not a key"}}), - source_location(inner_loc)); - } - lex_ws::invoke(inner_loc); - const auto close = lex_array_table_close::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "toml::parse_table_key: no `]]`", - {{source_location(inner_loc), "should be `]]`"}}), - source_location(inner_loc)); - } - - // after [[table.key]], newline or EOF(empty table) required. - if(loc.iter() != loc.end()) - { - using lex_newline_after_table_key = - sequence, maybe, lex_newline>; - const auto nl = lex_newline_after_table_key::invoke(loc); - if(!nl) - { - throw syntax_error(format_underline("toml::" - "parse_array_table_key: newline required after [[table.key]]", - {{source_location(loc), "expected newline"}}), - source_location(loc)); - } - } - return ok(std::make_pair(keys.unwrap().first, token.unwrap())); - } - else - { - return err(format_underline("toml::parse_array_table_key: " - "not a valid table key", {{source_location(loc), "here"}})); - } -} - -// parse table body (key-value pairs until the iter hits the next [tablekey]) -template -result -parse_ml_table(location& loc) -{ - using value_type = Value; - using table_type = typename value_type::table_type; - - const auto first = loc.iter(); - if(first == loc.end()) - { - return ok(table_type{}); - } - - // XXX at lest one newline is needed. - using skip_line = repeat< - sequence, maybe, lex_newline>, at_least<1>>; - skip_line::invoke(loc); - lex_ws::invoke(loc); - - table_type tab; - while(loc.iter() != loc.end()) - { - lex_ws::invoke(loc); - const auto before = loc.iter(); - if(const auto tmp = parse_array_table_key(loc)) // next table found - { - loc.reset(before); - return ok(tab); - } - if(const auto tmp = parse_table_key(loc)) // next table found - { - loc.reset(before); - return ok(tab); - } - - if(const auto kv = parse_key_value_pair(loc)) - { - const auto& kvpair = kv.unwrap(); - const std::vector& keys = kvpair.first.first; - const auto& key_reg = kvpair.first.second; - const value_type& val = kvpair.second; - const auto inserted = - insert_nested_key(tab, val, keys.begin(), keys.end(), key_reg); - if(!inserted) - { - return err(inserted.unwrap_err()); - } - } - else - { - return err(kv.unwrap_err()); - } - - // comment lines are skipped by the above function call. - // However, since the `skip_line` requires at least 1 newline, it fails - // if the file ends with ws and/or comment without newline. - // `skip_line` matches `ws? + comment? + newline`, not `ws` or `comment` - // itself. To skip the last ws and/or comment, call lexers. - // It does not matter if these fails, so the return value is discarded. - lex_ws::invoke(loc); - lex_comment::invoke(loc); - - // skip_line is (whitespace? comment? newline)_{1,}. multiple empty lines - // and comments after the last key-value pairs are allowed. - const auto newline = skip_line::invoke(loc); - if(!newline && loc.iter() != loc.end()) - { - const auto before2 = loc.iter(); - lex_ws::invoke(loc); // skip whitespace - const auto msg = format_underline("toml::parse_table: " - "invalid line format", {{source_location(loc), concat_to_string( - "expected newline, but got '", show_char(*loc.iter()), "'.")}}); - loc.reset(before2); - return err(msg); - } - - // the skip_lines only matches with lines that includes newline. - // to skip the last line that includes comment and/or whitespace - // but no newline, call them one more time. - lex_ws::invoke(loc); - lex_comment::invoke(loc); - } - return ok(tab); -} - -template -result parse_toml_file(location& loc) -{ - using value_type = Value; - using table_type = typename value_type::table_type; - - const auto first = loc.iter(); - if(first == loc.end()) - { - // For empty files, return an empty table with an empty region (zero-length). - // Without the region, error messages would miss the filename. - return ok(value_type(table_type{}, region(loc, first, first), {})); - } - - // put the first line as a region of a file - // Here first != loc.end(), so taking std::next is okay - const region file(loc, first, std::next(loc.iter())); - - // The first successive comments that are separated from the first value - // by an empty line are for a file itself. - // ```toml - // # this is a comment for a file. - // - // key = "the first value" - // ``` - // ```toml - // # this is a comment for "the first value". - // key = "the first value" - // ``` - std::vector comments; - using lex_first_comments = sequence< - repeat, lex_comment, lex_newline>, at_least<1>>, - sequence, lex_newline> - >; - if(const auto token = lex_first_comments::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - while(inner_loc.iter() != inner_loc.end()) - { - maybe::invoke(inner_loc); // remove ws if exists - if(lex_newline::invoke(inner_loc)) - { - assert(inner_loc.iter() == inner_loc.end()); - break; // empty line found. - } - auto com = lex_comment::invoke(inner_loc).unwrap().str(); - com.erase(com.begin()); // remove # sign - comments.push_back(std::move(com)); - lex_newline::invoke(inner_loc); - } - } - - table_type data; - // root object is also a table, but without [tablename] - if(const auto tab = parse_ml_table(loc)) - { - data = std::move(tab.unwrap()); - } - else // failed (empty table is regarded as success in parse_ml_table) - { - return err(tab.unwrap_err()); - } - while(loc.iter() != loc.end()) - { - // here, the region of [table] is regarded as the table-key because - // the table body is normally too big and it is not so informative - // if the first key-value pair of the table is shown in the error - // message. - if(const auto tabkey = parse_array_table_key(loc)) - { - const auto tab = parse_ml_table(loc); - if(!tab){return err(tab.unwrap_err());} - - const auto& tk = tabkey.unwrap(); - const auto& keys = tk.first; - const auto& reg = tk.second; - - const auto inserted = insert_nested_key(data, - value_type(tab.unwrap(), reg, reg.comments()), - keys.begin(), keys.end(), reg, - /*is_array_of_table=*/ true); - if(!inserted) {return err(inserted.unwrap_err());} - - continue; - } - if(const auto tabkey = parse_table_key(loc)) - { - const auto tab = parse_ml_table(loc); - if(!tab){return err(tab.unwrap_err());} - - const auto& tk = tabkey.unwrap(); - const auto& keys = tk.first; - const auto& reg = tk.second; - - const auto inserted = insert_nested_key(data, - value_type(tab.unwrap(), reg, reg.comments()), - keys.begin(), keys.end(), reg); - if(!inserted) {return err(inserted.unwrap_err());} - - continue; - } - return err(format_underline("toml::parse_toml_file: " - "unknown line appeared", {{source_location(loc), "unknown format"}})); - } - - return ok(Value(std::move(data), file, comments)); -} - -} // detail - -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value -parse(std::istream& is, const std::string& fname = "unknown file") -{ - using value_type = basic_value; - - const auto beg = is.tellg(); - is.seekg(0, std::ios::end); - const auto end = is.tellg(); - const auto fsize = end - beg; - is.seekg(beg); - - // read whole file as a sequence of char - assert(fsize >= 0); - std::vector letters(static_cast(fsize)); - is.read(letters.data(), fsize); - - // append LF. - // Although TOML does not require LF at the EOF, to make parsing logic - // simpler, we "normalize" the content by adding LF if it does not exist. - // It also checks if the last char is CR, to avoid changing the meaning. - // This is not the *best* way to deal with the last character, but is a - // simple and quick fix. - if(!letters.empty() && letters.back() != '\n' && letters.back() != '\r') - { - letters.push_back('\n'); - } - - detail::location loc(std::move(fname), std::move(letters)); - - // skip BOM if exists. - // XXX component of BOM (like 0xEF) exceeds the representable range of - // signed char, so on some (actually, most) of the environment, these cannot - // be compared to char. However, since we are always out of luck, we need to - // check our chars are equivalent to BOM. To do this, first we need to - // convert char to unsigned char to guarantee the comparability. - if(loc.source()->size() >= 3) - { - std::array BOM; - std::memcpy(BOM.data(), loc.source()->data(), 3); - if(BOM[0] == 0xEF && BOM[1] == 0xBB && BOM[2] == 0xBF) - { - loc.advance(3); // BOM found. skip. - } - } - - const auto data = detail::parse_toml_file(loc); - if(!data) - { - throw syntax_error(data.unwrap_err(), source_location(loc)); - } - return data.unwrap(); -} - -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value parse(const std::string& fname) -{ - std::ifstream ifs(fname.c_str(), std::ios_base::binary); - if(!ifs.good()) - { - throw std::runtime_error("toml::parse: file open error -> " + fname); - } - return parse(ifs, fname); -} - -#ifdef TOML11_HAS_STD_FILESYSTEM -// This function just forwards `parse("filename.toml")` to std::string version -// to avoid the ambiguity in overload resolution. -// -// Both std::string and std::filesystem::path are convertible from const char*. -// Without this, both parse(std::string) and parse(std::filesystem::path) -// matches to parse("filename.toml"). This breaks the existing code. -// -// This function exactly matches to the invocation with c-string. -// So this function is preferred than others and the ambiguity disappears. -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value parse(const char* fname) -{ - return parse(std::string(fname)); -} - -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value parse(const std::filesystem::path& fpath) -{ - std::ifstream ifs(fpath, std::ios_base::binary); - if(!ifs.good()) - { - throw std::runtime_error("toml::parse: file open error -> " + - fpath.string()); - } - return parse(ifs, fpath.string()); -} -#endif // TOML11_HAS_STD_FILESYSTEM - -} // toml -#endif// TOML11_PARSER_HPP diff -Nru nix-2.8.0/src/toml11/toml/region.hpp nix-2.16.1+dfsg/src/toml11/toml/region.hpp --- nix-2.8.0/src/toml11/toml/region.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/region.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,417 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_REGION_HPP -#define TOML11_REGION_HPP -#include -#include -#include -#include -#include -#include -#include -#include "color.hpp" - -namespace toml -{ -namespace detail -{ - -// helper function to avoid std::string(0, 'c') or std::string(iter, iter) -template -std::string make_string(Iterator first, Iterator last) -{ - if(first == last) {return "";} - return std::string(first, last); -} -inline std::string make_string(std::size_t len, char c) -{ - if(len == 0) {return "";} - return std::string(len, c); -} - -// region_base is a base class of location and region that are defined below. -// it will be used to generate better error messages. -struct region_base -{ - region_base() = default; - virtual ~region_base() = default; - region_base(const region_base&) = default; - region_base(region_base&& ) = default; - region_base& operator=(const region_base&) = default; - region_base& operator=(region_base&& ) = default; - - virtual bool is_ok() const noexcept {return false;} - virtual char front() const noexcept {return '\0';} - - virtual std::string str() const {return std::string("unknown region");} - virtual std::string name() const {return std::string("unknown file");} - virtual std::string line() const {return std::string("unknown line");} - virtual std::string line_num() const {return std::string("?");} - - // length of the region - virtual std::size_t size() const noexcept {return 0;} - // number of characters in the line before the region - virtual std::size_t before() const noexcept {return 0;} - // number of characters in the line after the region - virtual std::size_t after() const noexcept {return 0;} - - virtual std::vector comments() const {return {};} - // ```toml - // # comment_before - // key = "value" # comment_inline - // ``` -}; - -// location represents a position in a container, which contains a file content. -// it can be considered as a region that contains only one character. -// -// it contains pointer to the file content and iterator that points the current -// location. -struct location final : public region_base -{ - using const_iterator = typename std::vector::const_iterator; - using difference_type = typename const_iterator::difference_type; - using source_ptr = std::shared_ptr>; - - location(std::string source_name, std::vector cont) - : source_(std::make_shared>(std::move(cont))), - line_number_(1), source_name_(std::move(source_name)), iter_(source_->cbegin()) - {} - location(std::string source_name, const std::string& cont) - : source_(std::make_shared>(cont.begin(), cont.end())), - line_number_(1), source_name_(std::move(source_name)), iter_(source_->cbegin()) - {} - - location(const location&) = default; - location(location&&) = default; - location& operator=(const location&) = default; - location& operator=(location&&) = default; - ~location() = default; - - bool is_ok() const noexcept override {return static_cast(source_);} - char front() const noexcept override {return *iter_;} - - // this const prohibits codes like `++(loc.iter())`. - const const_iterator iter() const noexcept {return iter_;} - - const_iterator begin() const noexcept {return source_->cbegin();} - const_iterator end() const noexcept {return source_->cend();} - - // XXX `location::line_num()` used to be implemented using `std::count` to - // count a number of '\n'. But with a long toml file (typically, 10k lines), - // it becomes intolerably slow because each time it generates error messages, - // it counts '\n' from thousands of characters. To workaround it, I decided - // to introduce `location::line_number_` member variable and synchronize it - // to the location changes the point to look. So an overload of `iter()` - // which returns mutable reference is removed and `advance()`, `retrace()` - // and `reset()` is added. - void advance(difference_type n = 1) noexcept - { - this->line_number_ += static_cast( - std::count(this->iter_, std::next(this->iter_, n), '\n')); - this->iter_ += n; - return; - } - void retrace(difference_type n = 1) noexcept - { - this->line_number_ -= static_cast( - std::count(std::prev(this->iter_, n), this->iter_, '\n')); - this->iter_ -= n; - return; - } - void reset(const_iterator rollback) noexcept - { - // since c++11, std::distance works in both ways for random-access - // iterators and returns a negative value if `first > last`. - if(0 <= std::distance(rollback, this->iter_)) // rollback < iter - { - this->line_number_ -= static_cast( - std::count(rollback, this->iter_, '\n')); - } - else // iter < rollback [[unlikely]] - { - this->line_number_ += static_cast( - std::count(this->iter_, rollback, '\n')); - } - this->iter_ = rollback; - return; - } - - std::string str() const override {return make_string(1, *this->iter());} - std::string name() const override {return source_name_;} - - std::string line_num() const override - { - return std::to_string(this->line_number_); - } - - std::string line() const override - { - return make_string(this->line_begin(), this->line_end()); - } - - const_iterator line_begin() const noexcept - { - using reverse_iterator = std::reverse_iterator; - return std::find(reverse_iterator(this->iter()), - reverse_iterator(this->begin()), '\n').base(); - } - const_iterator line_end() const noexcept - { - return std::find(this->iter(), this->end(), '\n'); - } - - // location is always points a character. so the size is 1. - std::size_t size() const noexcept override - { - return 1u; - } - std::size_t before() const noexcept override - { - const auto sz = std::distance(this->line_begin(), this->iter()); - assert(sz >= 0); - return static_cast(sz); - } - std::size_t after() const noexcept override - { - const auto sz = std::distance(this->iter(), this->line_end()); - assert(sz >= 0); - return static_cast(sz); - } - - source_ptr const& source() const& noexcept {return source_;} - source_ptr&& source() && noexcept {return std::move(source_);} - - private: - - source_ptr source_; - std::size_t line_number_; - std::string source_name_; - const_iterator iter_; -}; - -// region represents a range in a container, which contains a file content. -// -// it contains pointer to the file content and iterator that points the first -// and last location. -struct region final : public region_base -{ - using const_iterator = typename std::vector::const_iterator; - using source_ptr = std::shared_ptr>; - - // delete default constructor. source_ never be null. - region() = delete; - - explicit region(const location& loc) - : source_(loc.source()), source_name_(loc.name()), - first_(loc.iter()), last_(loc.iter()) - {} - explicit region(location&& loc) - : source_(loc.source()), source_name_(loc.name()), - first_(loc.iter()), last_(loc.iter()) - {} - - region(const location& loc, const_iterator f, const_iterator l) - : source_(loc.source()), source_name_(loc.name()), first_(f), last_(l) - {} - region(location&& loc, const_iterator f, const_iterator l) - : source_(loc.source()), source_name_(loc.name()), first_(f), last_(l) - {} - - region(const region&) = default; - region(region&&) = default; - region& operator=(const region&) = default; - region& operator=(region&&) = default; - ~region() = default; - - region& operator+=(const region& other) - { - // different regions cannot be concatenated - assert(this->begin() == other.begin() && this->end() == other.end() && - this->last_ == other.first_); - - this->last_ = other.last_; - return *this; - } - - bool is_ok() const noexcept override {return static_cast(source_);} - char front() const noexcept override {return *first_;} - - std::string str() const override {return make_string(first_, last_);} - std::string line() const override - { - if(this->contain_newline()) - { - return make_string(this->line_begin(), - std::find(this->line_begin(), this->last(), '\n')); - } - return make_string(this->line_begin(), this->line_end()); - } - std::string line_num() const override - { - return std::to_string(1 + std::count(this->begin(), this->first(), '\n')); - } - - std::size_t size() const noexcept override - { - const auto sz = std::distance(first_, last_); - assert(sz >= 0); - return static_cast(sz); - } - std::size_t before() const noexcept override - { - const auto sz = std::distance(this->line_begin(), this->first()); - assert(sz >= 0); - return static_cast(sz); - } - std::size_t after() const noexcept override - { - const auto sz = std::distance(this->last(), this->line_end()); - assert(sz >= 0); - return static_cast(sz); - } - - bool contain_newline() const noexcept - { - return std::find(this->first(), this->last(), '\n') != this->last(); - } - - const_iterator line_begin() const noexcept - { - using reverse_iterator = std::reverse_iterator; - return std::find(reverse_iterator(this->first()), - reverse_iterator(this->begin()), '\n').base(); - } - const_iterator line_end() const noexcept - { - return std::find(this->last(), this->end(), '\n'); - } - - const_iterator begin() const noexcept {return source_->cbegin();} - const_iterator end() const noexcept {return source_->cend();} - const_iterator first() const noexcept {return first_;} - const_iterator last() const noexcept {return last_;} - - source_ptr const& source() const& noexcept {return source_;} - source_ptr&& source() && noexcept {return std::move(source_);} - - std::string name() const override {return source_name_;} - - std::vector comments() const override - { - // assuming the current region (`*this`) points a value. - // ```toml - // a = "value" - // ^^^^^^^- this region - // ``` - using rev_iter = std::reverse_iterator; - - std::vector com{}; - { - // find comments just before the current region. - // ```toml - // # this should be collected. - // # this also. - // a = value # not this. - // ``` - - // # this is a comment for `a`, not array elements. - // a = [1, 2, 3, 4, 5] - if(this->first() == std::find_if(this->line_begin(), this->first(), - [](const char c) noexcept -> bool {return c == '[' || c == '{';})) - { - auto iter = this->line_begin(); // points the first character - while(iter != this->begin()) - { - iter = std::prev(iter); - - // range [line_start, iter) represents the previous line - const auto line_start = std::find( - rev_iter(iter), rev_iter(this->begin()), '\n').base(); - const auto comment_found = std::find(line_start, iter, '#'); - if(comment_found == iter) - { - break; // comment not found. - } - - // exclude the following case. - // > a = "foo" # comment // <-- this is not a comment for b but a. - // > b = "current value" - if(std::all_of(line_start, comment_found, - [](const char c) noexcept -> bool { - return c == ' ' || c == '\t'; - })) - { - // unwrap the first '#' by std::next. - auto s = make_string(std::next(comment_found), iter); - if(!s.empty() && s.back() == '\r') {s.pop_back();} - com.push_back(std::move(s)); - } - else - { - break; - } - iter = line_start; - } - } - } - - if(com.size() > 1) - { - std::reverse(com.begin(), com.end()); - } - - { - // find comments just after the current region. - // ```toml - // # not this. - // a = value # this one. - // a = [ # not this (technically difficult) - // - // ] # and this. - // ``` - // The reason why it's difficult is that it requires parsing in the - // following case. - // ```toml - // a = [ 10 # this comment is for `10`. not for `a` but `a[0]`. - // # ... - // ] # this is apparently a comment for a. - // - // b = [ - // 3.14 ] # there is no way to add a comment to `3.14` currently. - // - // c = [ - // 3.14 # do this if you need a comment here. - // ] - // ``` - const auto comment_found = - std::find(this->last(), this->line_end(), '#'); - if(comment_found != this->line_end()) // '#' found - { - // table = {key = "value"} # what is this for? - // the above comment is not for "value", but {key="value"}. - if(comment_found == std::find_if(this->last(), comment_found, - [](const char c) noexcept -> bool { - return !(c == ' ' || c == '\t' || c == ','); - })) - { - // unwrap the first '#' by std::next. - auto s = make_string(std::next(comment_found), this->line_end()); - if(!s.empty() && s.back() == '\r') {s.pop_back();} - com.push_back(std::move(s)); - } - } - } - return com; - } - - private: - - source_ptr source_; - std::string source_name_; - const_iterator first_, last_; -}; - -} // detail -} // toml -#endif// TOML11_REGION_H diff -Nru nix-2.8.0/src/toml11/toml/result.hpp nix-2.16.1+dfsg/src/toml11/toml/result.hpp --- nix-2.8.0/src/toml11/toml/result.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/result.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,717 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_RESULT_HPP -#define TOML11_RESULT_HPP -#include "traits.hpp" -#include -#include -#include -#include -#include -#include -#include - -namespace toml -{ - -template -struct success -{ - using value_type = T; - value_type value; - - explicit success(const value_type& v) - noexcept(std::is_nothrow_copy_constructible::value) - : value(v) - {} - explicit success(value_type&& v) - noexcept(std::is_nothrow_move_constructible::value) - : value(std::move(v)) - {} - - template - explicit success(U&& v): value(std::forward(v)) {} - - template - explicit success(const success& v): value(v.value) {} - template - explicit success(success&& v): value(std::move(v.value)) {} - - ~success() = default; - success(const success&) = default; - success(success&&) = default; - success& operator=(const success&) = default; - success& operator=(success&&) = default; -}; - -template -struct failure -{ - using value_type = T; - value_type value; - - explicit failure(const value_type& v) - noexcept(std::is_nothrow_copy_constructible::value) - : value(v) - {} - explicit failure(value_type&& v) - noexcept(std::is_nothrow_move_constructible::value) - : value(std::move(v)) - {} - - template - explicit failure(U&& v): value(std::forward(v)) {} - - template - explicit failure(const failure& v): value(v.value) {} - template - explicit failure(failure&& v): value(std::move(v.value)) {} - - ~failure() = default; - failure(const failure&) = default; - failure(failure&&) = default; - failure& operator=(const failure&) = default; - failure& operator=(failure&&) = default; -}; - -template -success::type>::type> -ok(T&& v) -{ - return success< - typename std::remove_cv::type>::type - >(std::forward(v)); -} -template -failure::type>::type> -err(T&& v) -{ - return failure< - typename std::remove_cv::type>::type - >(std::forward(v)); -} - -inline success ok(const char* literal) -{ - return success(std::string(literal)); -} -inline failure err(const char* literal) -{ - return failure(std::string(literal)); -} - - -template -struct result -{ - using value_type = T; - using error_type = E; - using success_type = success; - using failure_type = failure; - - result(const success_type& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(s); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - result(const failure_type& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(f); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - result(success_type&& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - result(failure_type&& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - - template - result(const success& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(s.value); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - template - result(const failure& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(f.value); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - template - result(success&& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s.value)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - template - result(failure&& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f.value)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - - result& operator=(const success_type& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(s); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - result& operator=(const failure_type& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(f); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - result& operator=(success_type&& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - result& operator=(failure_type&& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - - template - result& operator=(const success& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(s.value); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - template - result& operator=(const failure& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(f.value); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - template - result& operator=(success&& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s.value)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - template - result& operator=(failure&& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f.value)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - - ~result() noexcept {this->cleanup();} - - result(const result& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - result(result&& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - - template - result(const result& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - template - result(result&& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - - result& operator=(const result& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - result& operator=(result&& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - - template - result& operator=(const result& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - template - result& operator=(result&& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - - bool is_ok() const noexcept {return is_ok_;} - bool is_err() const noexcept {return !is_ok_;} - - operator bool() const noexcept {return is_ok_;} - - value_type& unwrap() & - { - if(is_err()) - { - throw std::runtime_error("toml::result: bad unwrap: " + - format_error(this->as_err())); - } - return this->succ.value; - } - value_type const& unwrap() const& - { - if(is_err()) - { - throw std::runtime_error("toml::result: bad unwrap: " + - format_error(this->as_err())); - } - return this->succ.value; - } - value_type&& unwrap() && - { - if(is_err()) - { - throw std::runtime_error("toml::result: bad unwrap: " + - format_error(this->as_err())); - } - return std::move(this->succ.value); - } - - value_type& unwrap_or(value_type& opt) & - { - if(is_err()) {return opt;} - return this->succ.value; - } - value_type const& unwrap_or(value_type const& opt) const& - { - if(is_err()) {return opt;} - return this->succ.value; - } - value_type unwrap_or(value_type opt) && - { - if(is_err()) {return opt;} - return this->succ.value; - } - - error_type& unwrap_err() & - { - if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} - return this->fail.value; - } - error_type const& unwrap_err() const& - { - if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} - return this->fail.value; - } - error_type&& unwrap_err() && - { - if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} - return std::move(this->fail.value); - } - - value_type& as_ok() & noexcept {return this->succ.value;} - value_type const& as_ok() const& noexcept {return this->succ.value;} - value_type&& as_ok() && noexcept {return std::move(this->succ.value);} - - error_type& as_err() & noexcept {return this->fail.value;} - error_type const& as_err() const& noexcept {return this->fail.value;} - error_type&& as_err() && noexcept {return std::move(this->fail.value);} - - - // prerequisities - // F: T -> U - // retval: result - template - result, error_type> - map(F&& f) & - { - if(this->is_ok()){return ok(f(this->as_ok()));} - return err(this->as_err()); - } - template - result, error_type> - map(F&& f) const& - { - if(this->is_ok()){return ok(f(this->as_ok()));} - return err(this->as_err()); - } - template - result, error_type> - map(F&& f) && - { - if(this->is_ok()){return ok(f(std::move(this->as_ok())));} - return err(std::move(this->as_err())); - } - - // prerequisities - // F: E -> F - // retval: result - template - result> - map_err(F&& f) & - { - if(this->is_err()){return err(f(this->as_err()));} - return ok(this->as_ok()); - } - template - result> - map_err(F&& f) const& - { - if(this->is_err()){return err(f(this->as_err()));} - return ok(this->as_ok()); - } - template - result> - map_err(F&& f) && - { - if(this->is_err()){return err(f(std::move(this->as_err())));} - return ok(std::move(this->as_ok())); - } - - // prerequisities - // F: T -> U - // retval: U - template - detail::return_type_of_t - map_or_else(F&& f, U&& opt) & - { - if(this->is_err()){return std::forward(opt);} - return f(this->as_ok()); - } - template - detail::return_type_of_t - map_or_else(F&& f, U&& opt) const& - { - if(this->is_err()){return std::forward(opt);} - return f(this->as_ok()); - } - template - detail::return_type_of_t - map_or_else(F&& f, U&& opt) && - { - if(this->is_err()){return std::forward(opt);} - return f(std::move(this->as_ok())); - } - - // prerequisities - // F: E -> U - // retval: U - template - detail::return_type_of_t - map_err_or_else(F&& f, U&& opt) & - { - if(this->is_ok()){return std::forward(opt);} - return f(this->as_err()); - } - template - detail::return_type_of_t - map_err_or_else(F&& f, U&& opt) const& - { - if(this->is_ok()){return std::forward(opt);} - return f(this->as_err()); - } - template - detail::return_type_of_t - map_err_or_else(F&& f, U&& opt) && - { - if(this->is_ok()){return std::forward(opt);} - return f(std::move(this->as_err())); - } - - // prerequisities: - // F: func T -> U - // toml::err(error_type) should be convertible to U. - // normally, type U is another result and E is convertible to F - template - detail::return_type_of_t - and_then(F&& f) & - { - if(this->is_ok()){return f(this->as_ok());} - return err(this->as_err()); - } - template - detail::return_type_of_t - and_then(F&& f) const& - { - if(this->is_ok()){return f(this->as_ok());} - return err(this->as_err()); - } - template - detail::return_type_of_t - and_then(F&& f) && - { - if(this->is_ok()){return f(std::move(this->as_ok()));} - return err(std::move(this->as_err())); - } - - // prerequisities: - // F: func E -> U - // toml::ok(value_type) should be convertible to U. - // normally, type U is another result and T is convertible to S - template - detail::return_type_of_t - or_else(F&& f) & - { - if(this->is_err()){return f(this->as_err());} - return ok(this->as_ok()); - } - template - detail::return_type_of_t - or_else(F&& f) const& - { - if(this->is_err()){return f(this->as_err());} - return ok(this->as_ok()); - } - template - detail::return_type_of_t - or_else(F&& f) && - { - if(this->is_err()){return f(std::move(this->as_err()));} - return ok(std::move(this->as_ok())); - } - - // if *this is error, returns *this. otherwise, returns other. - result and_other(const result& other) const& - { - return this->is_err() ? *this : other; - } - result and_other(result&& other) && - { - return this->is_err() ? std::move(*this) : std::move(other); - } - - // if *this is okay, returns *this. otherwise, returns other. - result or_other(const result& other) const& - { - return this->is_ok() ? *this : other; - } - result or_other(result&& other) && - { - return this->is_ok() ? std::move(*this) : std::move(other); - } - - void swap(result& other) - { - result tmp(std::move(*this)); - *this = std::move(other); - other = std::move(tmp); - return ; - } - - private: - - static std::string format_error(std::exception const& excpt) - { - return std::string(excpt.what()); - } - template::value, std::nullptr_t>::type = nullptr> - static std::string format_error(U const& others) - { - std::ostringstream oss; oss << others; - return oss.str(); - } - - void cleanup() noexcept - { - if(this->is_ok_) {this->succ.~success_type();} - else {this->fail.~failure_type();} - return; - } - - private: - - bool is_ok_; - union - { - success_type succ; - failure_type fail; - }; -}; - -template -void swap(result& lhs, result& rhs) -{ - lhs.swap(rhs); - return; -} - -// this might be confusing because it eagerly evaluated, while in the other -// cases operator && and || are short-circuited. -// -// template -// inline result -// operator&&(const result& lhs, const result& rhs) noexcept -// { -// return lhs.is_ok() ? rhs : lhs; -// } -// -// template -// inline result -// operator||(const result& lhs, const result& rhs) noexcept -// { -// return lhs.is_ok() ? lhs : rhs; -// } - -// ---------------------------------------------------------------------------- -// re-use result as a optional with none_t - -namespace detail -{ -struct none_t {}; -inline bool operator==(const none_t&, const none_t&) noexcept {return true;} -inline bool operator!=(const none_t&, const none_t&) noexcept {return false;} -inline bool operator< (const none_t&, const none_t&) noexcept {return false;} -inline bool operator<=(const none_t&, const none_t&) noexcept {return true;} -inline bool operator> (const none_t&, const none_t&) noexcept {return false;} -inline bool operator>=(const none_t&, const none_t&) noexcept {return true;} -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const none_t&) -{ - os << "none"; - return os; -} -inline failure none() noexcept {return failure{none_t{}};} -} // detail -} // toml11 -#endif// TOML11_RESULT_H diff -Nru nix-2.8.0/src/toml11/toml/serializer.hpp nix-2.16.1+dfsg/src/toml11/toml/serializer.hpp --- nix-2.8.0/src/toml11/toml/serializer.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/serializer.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,922 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_SERIALIZER_HPP -#define TOML11_SERIALIZER_HPP -#include -#include - -#include - -#include "lexer.hpp" -#include "value.hpp" - -namespace toml -{ - -// This function serialize a key. It checks a string is a bare key and -// escapes special characters if the string is not compatible to a bare key. -// ```cpp -// std::string k("non.bare.key"); // the key itself includes `.`s. -// std::string formatted = toml::format_key(k); -// assert(formatted == "\"non.bare.key\""); -// ``` -// -// This function is exposed to make it easy to write a user-defined serializer. -// Since toml restricts characters available in a bare key, generally a string -// should be escaped. But checking whether a string needs to be surrounded by -// a `"` and escaping some special character is boring. -template -std::basic_string -format_key(const std::basic_string& k) -{ - if(k.empty()) - { - return std::string("\"\""); - } - - // check the key can be a bare (unquoted) key - detail::location loc(k, std::vector(k.begin(), k.end())); - detail::lex_unquoted_key::invoke(loc); - if(loc.iter() == loc.end()) - { - return k; // all the tokens are consumed. the key is unquoted-key. - } - - //if it includes special characters, then format it in a "quoted" key. - std::basic_string serialized("\""); - for(const char c : k) - { - switch(c) - { - case '\\': {serialized += "\\\\"; break;} - case '\"': {serialized += "\\\""; break;} - case '\b': {serialized += "\\b"; break;} - case '\t': {serialized += "\\t"; break;} - case '\f': {serialized += "\\f"; break;} - case '\n': {serialized += "\\n"; break;} - case '\r': {serialized += "\\r"; break;} - default : {serialized += c; break;} - } - } - serialized += "\""; - return serialized; -} - -template -std::basic_string -format_keys(const std::vector>& keys) -{ - if(keys.empty()) - { - return std::string("\"\""); - } - - std::basic_string serialized; - for(const auto& ky : keys) - { - serialized += format_key(ky); - serialized += charT('.'); - } - serialized.pop_back(); // remove the last dot '.' - return serialized; -} - -template -struct serializer -{ - static_assert(detail::is_basic_value::value, - "toml::serializer is for toml::value and its variants, " - "toml::basic_value<...>."); - - using value_type = Value; - using key_type = typename value_type::key_type ; - using comment_type = typename value_type::comment_type ; - using boolean_type = typename value_type::boolean_type ; - using integer_type = typename value_type::integer_type ; - using floating_type = typename value_type::floating_type ; - using string_type = typename value_type::string_type ; - using local_time_type = typename value_type::local_time_type ; - using local_date_type = typename value_type::local_date_type ; - using local_datetime_type = typename value_type::local_datetime_type ; - using offset_datetime_type = typename value_type::offset_datetime_type; - using array_type = typename value_type::array_type ; - using table_type = typename value_type::table_type ; - - serializer(const std::size_t w = 80u, - const int float_prec = std::numeric_limits::max_digits10, - const bool can_be_inlined = false, - const bool no_comment = false, - std::vector ks = {}, - const bool value_has_comment = false) - : can_be_inlined_(can_be_inlined), no_comment_(no_comment), - value_has_comment_(value_has_comment && !no_comment), - float_prec_(float_prec), width_(w), keys_(std::move(ks)) - {} - ~serializer() = default; - - std::string operator()(const boolean_type& b) const - { - return b ? "true" : "false"; - } - std::string operator()(const integer_type i) const - { - return std::to_string(i); - } - std::string operator()(const floating_type f) const - { - if(std::isnan(f)) - { - if(std::signbit(f)) - { - return std::string("-nan"); - } - else - { - return std::string("nan"); - } - } - else if(!std::isfinite(f)) - { - if(std::signbit(f)) - { - return std::string("-inf"); - } - else - { - return std::string("inf"); - } - } - - const auto fmt = "%.*g"; - const auto bsz = std::snprintf(nullptr, 0, fmt, this->float_prec_, f); - // +1 for null character(\0) - std::vector buf(static_cast(bsz + 1), '\0'); - std::snprintf(buf.data(), buf.size(), fmt, this->float_prec_, f); - - std::string token(buf.begin(), std::prev(buf.end())); - if(!token.empty() && token.back() == '.') // 1. => 1.0 - { - token += '0'; - } - - const auto e = std::find_if( - token.cbegin(), token.cend(), [](const char c) noexcept -> bool { - return c == 'e' || c == 'E'; - }); - const auto has_exponent = (token.cend() != e); - const auto has_fraction = (token.cend() != std::find( - token.cbegin(), token.cend(), '.')); - - if(!has_exponent && !has_fraction) - { - // the resulting value does not have any float specific part! - token += ".0"; - } - return token; - } - std::string operator()(const string_type& s) const - { - if(s.kind == string_t::basic) - { - if((std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || - std::find(s.str.cbegin(), s.str.cend(), '\"') != s.str.cend()) && - this->width_ != (std::numeric_limits::max)()) - { - // if linefeed or double-quote is contained, - // make it multiline basic string. - const auto escaped = this->escape_ml_basic_string(s.str); - std::string open("\"\"\""); - std::string close("\"\"\""); - if(escaped.find('\n') != std::string::npos || - this->width_ < escaped.size() + 6) - { - // if the string body contains newline or is enough long, - // add newlines after and before delimiters. - open += "\n"; - close = std::string("\\\n") + close; - } - return open + escaped + close; - } - - // no linefeed. try to make it oneline-string. - std::string oneline = this->escape_basic_string(s.str); - if(oneline.size() + 2 < width_ || width_ < 2) - { - const std::string quote("\""); - return quote + oneline + quote; - } - - // the line is too long compared to the specified width. - // split it into multiple lines. - std::string token("\"\"\"\n"); - while(!oneline.empty()) - { - if(oneline.size() < width_) - { - token += oneline; - oneline.clear(); - } - else if(oneline.at(width_-2) == '\\') - { - token += oneline.substr(0, width_-2); - token += "\\\n"; - oneline.erase(0, width_-2); - } - else - { - token += oneline.substr(0, width_-1); - token += "\\\n"; - oneline.erase(0, width_-1); - } - } - return token + std::string("\\\n\"\"\""); - } - else // the string `s` is literal-string. - { - if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || - std::find(s.str.cbegin(), s.str.cend(), '\'') != s.str.cend() ) - { - std::string open("'''"); - if(this->width_ + 6 < s.str.size()) - { - open += '\n'; // the first newline is ignored by TOML spec - } - const std::string close("'''"); - return open + s.str + close; - } - else - { - const std::string quote("'"); - return quote + s.str + quote; - } - } - } - - std::string operator()(const local_date_type& d) const - { - std::ostringstream oss; - oss << d; - return oss.str(); - } - std::string operator()(const local_time_type& t) const - { - std::ostringstream oss; - oss << t; - return oss.str(); - } - std::string operator()(const local_datetime_type& dt) const - { - std::ostringstream oss; - oss << dt; - return oss.str(); - } - std::string operator()(const offset_datetime_type& odt) const - { - std::ostringstream oss; - oss << odt; - return oss.str(); - } - - std::string operator()(const array_type& v) const - { - if(v.empty()) - { - return std::string("[]"); - } - if(this->is_array_of_tables(v)) - { - return make_array_of_tables(v); - } - - // not an array of tables. normal array. - // first, try to make it inline if none of the elements have a comment. - if( ! this->has_comment_inside(v)) - { - const auto inl = this->make_inline_array(v); - if(inl.size() < this->width_ && - std::find(inl.cbegin(), inl.cend(), '\n') == inl.cend()) - { - return inl; - } - } - - // if the length exceeds this->width_, print multiline array. - // key = [ - // # ... - // 42, - // ... - // ] - std::string token; - std::string current_line; - token += "[\n"; - for(const auto& item : v) - { - if( ! item.comments().empty() && !no_comment_) - { - // if comment exists, the element must be the only element in the line. - // e.g. the following is not allowed. - // ```toml - // array = [ - // # comment for what? - // 1, 2, 3, 4, 5 - // ] - // ``` - if(!current_line.empty()) - { - if(current_line.back() != '\n') - { - current_line += '\n'; - } - token += current_line; - current_line.clear(); - } - for(const auto& c : item.comments()) - { - token += '#'; - token += c; - token += '\n'; - } - token += toml::visit(*this, item); - if(!token.empty() && token.back() == '\n') {token.pop_back();} - token += ",\n"; - continue; - } - std::string next_elem; - if(item.is_table()) - { - serializer ser(*this); - ser.can_be_inlined_ = true; - ser.width_ = (std::numeric_limits::max)(); - next_elem += toml::visit(ser, item); - } - else - { - next_elem += toml::visit(*this, item); - } - - // comma before newline. - if(!next_elem.empty() && next_elem.back() == '\n') {next_elem.pop_back();} - - // if current line does not exceeds the width limit, continue. - if(current_line.size() + next_elem.size() + 1 < this->width_) - { - current_line += next_elem; - current_line += ','; - } - else if(current_line.empty()) - { - // if current line was empty, force put the next_elem because - // next_elem is not splittable - token += next_elem; - token += ",\n"; - // current_line is kept empty - } - else // reset current_line - { - assert(current_line.back() == ','); - token += current_line; - token += '\n'; - current_line = next_elem; - current_line += ','; - } - } - if(!current_line.empty()) - { - if(!current_line.empty() && current_line.back() != '\n') - { - current_line += '\n'; - } - token += current_line; - } - token += "]\n"; - return token; - } - - // templatize for any table-like container - std::string operator()(const table_type& v) const - { - // if an element has a comment, then it can't be inlined. - // table = {# how can we write a comment for this? key = "value"} - if(this->can_be_inlined_ && !(this->has_comment_inside(v))) - { - std::string token; - if(!this->keys_.empty()) - { - token += format_key(this->keys_.back()); - token += " = "; - } - token += this->make_inline_table(v); - if(token.size() < this->width_ && - token.end() == std::find(token.begin(), token.end(), '\n')) - { - return token; - } - } - - std::string token; - if(!keys_.empty()) - { - token += '['; - token += format_keys(keys_); - token += "]\n"; - } - token += this->make_multiline_table(v); - return token; - } - - private: - - std::string escape_basic_string(const std::string& s) const - { - //XXX assuming `s` is a valid utf-8 sequence. - std::string retval; - for(const char c : s) - { - switch(c) - { - case '\\': {retval += "\\\\"; break;} - case '\"': {retval += "\\\""; break;} - case '\b': {retval += "\\b"; break;} - case '\t': {retval += "\\t"; break;} - case '\f': {retval += "\\f"; break;} - case '\n': {retval += "\\n"; break;} - case '\r': {retval += "\\r"; break;} - default : - { - if((0x00 <= c && c <= 0x08) || (0x0A <= c && c <= 0x1F) || c == 0x7F) - { - retval += "\\u00"; - retval += char(48 + (c / 16)); - retval += char((c % 16 < 10 ? 48 : 55) + (c % 16)); - } - else - { - retval += c; - } - } - } - } - return retval; - } - - std::string escape_ml_basic_string(const std::string& s) const - { - std::string retval; - for(auto i=s.cbegin(), e=s.cend(); i!=e; ++i) - { - switch(*i) - { - case '\\': {retval += "\\\\"; break;} - // One or two consecutive "s are allowed. - // Later we will check there are no three consecutive "s. - // case '\"': {retval += "\\\""; break;} - case '\b': {retval += "\\b"; break;} - case '\t': {retval += "\\t"; break;} - case '\f': {retval += "\\f"; break;} - case '\n': {retval += "\n"; break;} - case '\r': - { - if(std::next(i) != e && *std::next(i) == '\n') - { - retval += "\r\n"; - ++i; - } - else - { - retval += "\\r"; - } - break; - } - default : - { - const auto c = *i; - if((0x00 <= c && c <= 0x08) || (0x0A <= c && c <= 0x1F) || c == 0x7F) - { - retval += "\\u00"; - retval += char(48 + (c / 16)); - retval += char((c % 16 < 10 ? 48 : 55) + (c % 16)); - } - else - { - retval += c; - } - } - - } - } - // Only 1 or 2 consecutive `"`s are allowed in multiline basic string. - // 3 consecutive `"`s are considered as a closing delimiter. - // We need to check if there are 3 or more consecutive `"`s and insert - // backslash to break them down into several short `"`s like the `str6` - // in the following example. - // ```toml - // str4 = """Here are two quotation marks: "". Simple enough.""" - // # str5 = """Here are three quotation marks: """.""" # INVALID - // str5 = """Here are three quotation marks: ""\".""" - // str6 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\".""" - // ``` - auto found_3_quotes = retval.find("\"\"\""); - while(found_3_quotes != std::string::npos) - { - retval.replace(found_3_quotes, 3, "\"\"\\\""); - found_3_quotes = retval.find("\"\"\""); - } - return retval; - } - - // if an element of a table or an array has a comment, it cannot be inlined. - bool has_comment_inside(const array_type& a) const noexcept - { - // if no_comment is set, comments would not be written. - if(this->no_comment_) {return false;} - - for(const auto& v : a) - { - if(!v.comments().empty()) {return true;} - } - return false; - } - bool has_comment_inside(const table_type& t) const noexcept - { - // if no_comment is set, comments would not be written. - if(this->no_comment_) {return false;} - - for(const auto& kv : t) - { - if(!kv.second.comments().empty()) {return true;} - } - return false; - } - - std::string make_inline_array(const array_type& v) const - { - assert(!has_comment_inside(v)); - std::string token; - token += '['; - bool is_first = true; - for(const auto& item : v) - { - if(is_first) {is_first = false;} else {token += ',';} - token += visit(serializer( - (std::numeric_limits::max)(), this->float_prec_, - /* inlined */ true, /*no comment*/ false, /*keys*/ {}, - /*has_comment*/ !item.comments().empty()), item); - } - token += ']'; - return token; - } - - std::string make_inline_table(const table_type& v) const - { - assert(!has_comment_inside(v)); - assert(this->can_be_inlined_); - std::string token; - token += '{'; - bool is_first = true; - for(const auto& kv : v) - { - // in inline tables, trailing comma is not allowed (toml-lang #569). - if(is_first) {is_first = false;} else {token += ',';} - token += format_key(kv.first); - token += '='; - token += visit(serializer( - (std::numeric_limits::max)(), this->float_prec_, - /* inlined */ true, /*no comment*/ false, /*keys*/ {}, - /*has_comment*/ !kv.second.comments().empty()), kv.second); - } - token += '}'; - return token; - } - - std::string make_multiline_table(const table_type& v) const - { - std::string token; - - // print non-table elements first. - // ```toml - // [foo] # a table we're writing now here - // key = "value" # <- non-table element, "key" - // # ... - // [foo.bar] # <- table element, "bar" - // ``` - // because after printing [foo.bar], the remaining non-table values will - // be assigned into [foo.bar], not [foo]. Those values should be printed - // earlier. - for(const auto& kv : v) - { - if(kv.second.is_table() || is_array_of_tables(kv.second)) - { - continue; - } - - token += write_comments(kv.second); - - const auto key_and_sep = format_key(kv.first) + " = "; - const auto residual_width = (this->width_ > key_and_sep.size()) ? - this->width_ - key_and_sep.size() : 0; - token += key_and_sep; - token += visit(serializer(residual_width, this->float_prec_, - /*can be inlined*/ true, /*no comment*/ false, /*keys*/ {}, - /*has_comment*/ !kv.second.comments().empty()), kv.second); - - if(token.back() != '\n') - { - token += '\n'; - } - } - - // normal tables / array of tables - - // after multiline table appeared, the other tables cannot be inline - // because the table would be assigned into the table. - // [foo] - // ... - // bar = {...} # <- bar will be a member of [foo]. - bool multiline_table_printed = false; - for(const auto& kv : v) - { - if(!kv.second.is_table() && !is_array_of_tables(kv.second)) - { - continue; // other stuff are already serialized. skip them. - } - - std::vector ks(this->keys_); - ks.push_back(kv.first); - - auto tmp = visit(serializer(this->width_, this->float_prec_, - !multiline_table_printed, this->no_comment_, ks, - /*has_comment*/ !kv.second.comments().empty()), kv.second); - - // If it is the first time to print a multi-line table, it would be - // helpful to separate normal key-value pair and subtables by a - // newline. - // (this checks if the current key-value pair contains newlines. - // but it is not perfect because multi-line string can also contain - // a newline. in such a case, an empty line will be written) TODO - if((!multiline_table_printed) && - std::find(tmp.cbegin(), tmp.cend(), '\n') != tmp.cend()) - { - multiline_table_printed = true; - token += '\n'; // separate key-value pairs and subtables - - token += write_comments(kv.second); - token += tmp; - - // care about recursive tables (all tables in each level prints - // newline and there will be a full of newlines) - if(tmp.substr(tmp.size() - 2, 2) != "\n\n" && - tmp.substr(tmp.size() - 4, 4) != "\r\n\r\n" ) - { - token += '\n'; - } - } - else - { - token += write_comments(kv.second); - token += tmp; - token += '\n'; - } - } - return token; - } - - std::string make_array_of_tables(const array_type& v) const - { - // if it's not inlined, we need to add `[[table.key]]`. - // but if it can be inlined, we can format it as the following. - // ``` - // table.key = [ - // {...}, - // # comment - // {...}, - // ] - // ``` - // This function checks if inlinization is possible or not, and then - // format the array-of-tables in a proper way. - // - // Note about comments: - // - // If the array itself has a comment (value_has_comment_ == true), we - // should try to make it inline. - // ```toml - // # comment about array - // array = [ - // # comment about table element - // {of = "table"} - // ] - // ``` - // If it is formatted as a multiline table, the two comments becomes - // indistinguishable. - // ```toml - // # comment about array - // # comment about table element - // [[array]] - // of = "table" - // ``` - // So we need to try to make it inline, and it force-inlines regardless - // of the line width limit. - // It may fail if the element of a table has comment. In that case, - // the array-of-tables will be formatted as a multiline table. - if(this->can_be_inlined_ || this->value_has_comment_) - { - std::string token; - if(!keys_.empty()) - { - token += format_key(keys_.back()); - token += " = "; - } - - bool failed = false; - token += "[\n"; - for(const auto& item : v) - { - // if an element of the table has a comment, the table - // cannot be inlined. - if(this->has_comment_inside(item.as_table())) - { - failed = true; - break; - } - // write comments for the table itself - token += write_comments(item); - - const auto t = this->make_inline_table(item.as_table()); - - if(t.size() + 1 > width_ || // +1 for the last comma {...}, - std::find(t.cbegin(), t.cend(), '\n') != t.cend()) - { - // if the value itself has a comment, ignore the line width limit - if( ! this->value_has_comment_) - { - failed = true; - break; - } - } - token += t; - token += ",\n"; - } - - if( ! failed) - { - token += "]\n"; - return token; - } - // if failed, serialize them as [[array.of.tables]]. - } - - std::string token; - for(const auto& item : v) - { - token += write_comments(item); - token += "[["; - token += format_keys(keys_); - token += "]]\n"; - token += this->make_multiline_table(item.as_table()); - } - return token; - } - - std::string write_comments(const value_type& v) const - { - std::string retval; - if(this->no_comment_) {return retval;} - - for(const auto& c : v.comments()) - { - retval += '#'; - retval += c; - retval += '\n'; - } - return retval; - } - - bool is_array_of_tables(const value_type& v) const - { - if(!v.is_array() || v.as_array().empty()) {return false;} - return is_array_of_tables(v.as_array()); - } - bool is_array_of_tables(const array_type& v) const - { - // Since TOML v0.5.0, heterogeneous arrays are allowed. So we need to - // check all the element in an array to check if the array is an array - // of tables. - return std::all_of(v.begin(), v.end(), [](const value_type& elem) { - return elem.is_table(); - }); - } - - private: - - bool can_be_inlined_; - bool no_comment_; - bool value_has_comment_; - int float_prec_; - std::size_t width_; - std::vector keys_; -}; - -template class M, template class V> -std::string -format(const basic_value& v, std::size_t w = 80u, - int fprec = std::numeric_limits::max_digits10, - bool no_comment = false, bool force_inline = false) -{ - using value_type = basic_value; - // if value is a table, it is considered to be a root object. - // the root object can't be an inline table. - if(v.is_table()) - { - std::ostringstream oss; - if(!v.comments().empty()) - { - oss << v.comments(); - oss << '\n'; // to split the file comment from the first element - } - const auto serialized = visit(serializer(w, fprec, false, no_comment), v); - oss << serialized; - return oss.str(); - } - return visit(serializer(w, fprec, force_inline), v); -} - -namespace detail -{ -template -int comment_index(std::basic_ostream&) -{ - static const int index = std::ios_base::xalloc(); - return index; -} -} // detail - -template -std::basic_ostream& -nocomment(std::basic_ostream& os) -{ - // by default, it is zero. and by default, it shows comments. - os.iword(detail::comment_index(os)) = 1; - return os; -} - -template -std::basic_ostream& -showcomment(std::basic_ostream& os) -{ - // by default, it is zero. and by default, it shows comments. - os.iword(detail::comment_index(os)) = 0; - return os; -} - -template class M, template class V> -std::basic_ostream& -operator<<(std::basic_ostream& os, const basic_value& v) -{ - using value_type = basic_value; - - // get status of std::setw(). - const auto w = static_cast(os.width()); - const int fprec = static_cast(os.precision()); - os.width(0); - - // by default, iword is initialized by 0. And by default, toml11 outputs - // comments. So `0` means showcomment. 1 means nocommnet. - const bool no_comment = (1 == os.iword(detail::comment_index(os))); - - if(!no_comment && v.is_table() && !v.comments().empty()) - { - os << v.comments(); - os << '\n'; // to split the file comment from the first element - } - // the root object can't be an inline table. so pass `false`. - const auto serialized = visit(serializer(w, fprec, no_comment, false), v); - os << serialized; - - // if v is a non-table value, and has only one comment, then - // put a comment just after a value. in the following way. - // - // ```toml - // key = "value" # comment. - // ``` - // - // Since the top-level toml object is a table, one who want to put a - // non-table toml value must use this in a following way. - // - // ```cpp - // toml::value v; - // std::cout << "user-defined-key = " << v << std::endl; - // ``` - // - // In this case, it is impossible to put comments before key-value pair. - // The only way to preserve comments is to put all of them after a value. - if(!no_comment && !v.is_table() && !v.comments().empty()) - { - os << " #"; - for(const auto& c : v.comments()) {os << c;} - } - return os; -} - -} // toml -#endif// TOML11_SERIALIZER_HPP diff -Nru nix-2.8.0/src/toml11/toml/source_location.hpp nix-2.16.1+dfsg/src/toml11/toml/source_location.hpp --- nix-2.8.0/src/toml11/toml/source_location.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/source_location.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,233 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_SOURCE_LOCATION_HPP -#define TOML11_SOURCE_LOCATION_HPP -#include -#include - -#include "region.hpp" - -namespace toml -{ - -// A struct to contain location in a toml file. -// The interface imitates std::experimental::source_location, -// but not completely the same. -// -// It would be constructed by toml::value. It can be used to generate -// user-defined error messages. -// -// - std::uint_least32_t line() const noexcept -// - returns the line number where the region is on. -// - std::uint_least32_t column() const noexcept -// - returns the column number where the region starts. -// - std::uint_least32_t region() const noexcept -// - returns the size of the region. -// -// +-- line() +-- region of interest (region() == 9) -// v .---+---. -// 12 | value = "foo bar" -// ^ -// +-- column() -// -// - std::string const& file_name() const noexcept; -// - name of the file. -// - std::string const& line_str() const noexcept; -// - the whole line that contains the region of interest. -// -struct source_location -{ - public: - - source_location() - : line_num_(1), column_num_(1), region_size_(1), - file_name_("unknown file"), line_str_("") - {} - - explicit source_location(const detail::region_base* reg) - : line_num_(1), column_num_(1), region_size_(1), - file_name_("unknown file"), line_str_("") - { - if(reg) - { - if(reg->line_num() != detail::region_base().line_num()) - { - line_num_ = static_cast( - std::stoul(reg->line_num())); - } - column_num_ = static_cast(reg->before() + 1); - region_size_ = static_cast(reg->size()); - file_name_ = reg->name(); - line_str_ = reg->line(); - } - } - - explicit source_location(const detail::region& reg) - : line_num_(static_cast(std::stoul(reg.line_num()))), - column_num_(static_cast(reg.before() + 1)), - region_size_(static_cast(reg.size())), - file_name_(reg.name()), - line_str_ (reg.line()) - {} - explicit source_location(const detail::location& loc) - : line_num_(static_cast(std::stoul(loc.line_num()))), - column_num_(static_cast(loc.before() + 1)), - region_size_(static_cast(loc.size())), - file_name_(loc.name()), - line_str_ (loc.line()) - {} - - ~source_location() = default; - source_location(source_location const&) = default; - source_location(source_location &&) = default; - source_location& operator=(source_location const&) = default; - source_location& operator=(source_location &&) = default; - - std::uint_least32_t line() const noexcept {return line_num_;} - std::uint_least32_t column() const noexcept {return column_num_;} - std::uint_least32_t region() const noexcept {return region_size_;} - - std::string const& file_name() const noexcept {return file_name_;} - std::string const& line_str() const noexcept {return line_str_;} - - private: - - std::uint_least32_t line_num_; - std::uint_least32_t column_num_; - std::uint_least32_t region_size_; - std::string file_name_; - std::string line_str_; -}; - -namespace detail -{ - -// internal error message generation. -inline std::string format_underline(const std::string& message, - const std::vector>& loc_com, - const std::vector& helps = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - std::size_t line_num_width = 0; - for(const auto& lc : loc_com) - { - std::uint_least32_t line = lc.first.line(); - std::size_t digit = 0; - while(line != 0) - { - line /= 10; - digit += 1; - } - line_num_width = (std::max)(line_num_width, digit); - } - // 1 is the minimum width - line_num_width = std::max(line_num_width, 1); - - std::ostringstream retval; - - if(colorize) - { - retval << color::colorize; // turn on ANSI color - } - - // XXX - // Here, before `colorize` support, it does not output `[error]` prefix - // automatically. So some user may output it manually and this change may - // duplicate the prefix. To avoid it, check the first 7 characters and - // if it is "[error]", it removes that part from the message shown. - if(message.size() > 7 && message.substr(0, 7) == "[error]") - { - retval << color::bold << color::red << "[error]" << color::reset - << color::bold << message.substr(7) << color::reset << '\n'; - } - else - { - retval << color::bold << color::red << "[error] " << color::reset - << color::bold << message << color::reset << '\n'; - } - - const auto format_one_location = [line_num_width] - (std::ostringstream& oss, - const source_location& loc, const std::string& comment) -> void - { - oss << ' ' << color::bold << color::blue - << std::setw(static_cast(line_num_width)) - << std::right << loc.line() << " | " << color::reset - << loc.line_str() << '\n'; - - oss << make_string(line_num_width + 1, ' ') - << color::bold << color::blue << " | " << color::reset - << make_string(loc.column()-1 /*1-origin*/, ' '); - - if(loc.region() == 1) - { - // invalid - // ^------ - oss << color::bold << color::red << "^---" << color::reset; - } - else - { - // invalid - // ~~~~~~~ - const auto underline_len = (std::min)( - static_cast(loc.region()), loc.line_str().size()); - oss << color::bold << color::red - << make_string(underline_len, '~') << color::reset; - } - oss << ' '; - oss << comment; - return; - }; - - assert(!loc_com.empty()); - - // --> example.toml - // | - retval << color::bold << color::blue << " --> " << color::reset - << loc_com.front().first.file_name() << '\n'; - retval << make_string(line_num_width + 1, ' ') - << color::bold << color::blue << " |\n" << color::reset; - // 1 | key value - // | ^--- missing = - format_one_location(retval, loc_com.front().first, loc_com.front().second); - - // process the rest of the locations - for(std::size_t i=1; i filename.toml" again - { - retval << color::bold << color::blue << " --> " << color::reset - << curr.first.file_name() << '\n'; - retval << make_string(line_num_width + 1, ' ') - << color::bold << color::blue << " |\n" << color::reset; - } - - format_one_location(retval, curr.first, curr.second); - } - - if(!helps.empty()) - { - retval << '\n'; - retval << make_string(line_num_width + 1, ' '); - retval << color::bold << color::blue << " |" << color::reset; - for(const auto& help : helps) - { - retval << color::bold << "\nHint: " << color::reset; - retval << help; - } - } - return retval.str(); -} - -} // detail -} // toml -#endif// TOML11_SOURCE_LOCATION_HPP diff -Nru nix-2.8.0/src/toml11/toml/storage.hpp nix-2.16.1+dfsg/src/toml11/toml/storage.hpp --- nix-2.8.0/src/toml11/toml/storage.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/storage.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,43 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_STORAGE_HPP -#define TOML11_STORAGE_HPP -#include "utility.hpp" - -namespace toml -{ -namespace detail -{ - -// this contains pointer and deep-copy the content if copied. -// to avoid recursive pointer. -template -struct storage -{ - using value_type = T; - - explicit storage(value_type const& v): ptr(toml::make_unique(v)) {} - explicit storage(value_type&& v): ptr(toml::make_unique(std::move(v))) {} - ~storage() = default; - storage(const storage& rhs): ptr(toml::make_unique(*rhs.ptr)) {} - storage& operator=(const storage& rhs) - { - this->ptr = toml::make_unique(*rhs.ptr); - return *this; - } - storage(storage&&) = default; - storage& operator=(storage&&) = default; - - bool is_ok() const noexcept {return static_cast(ptr);} - - value_type& value() & noexcept {return *ptr;} - value_type const& value() const& noexcept {return *ptr;} - value_type&& value() && noexcept {return std::move(*ptr);} - - private: - std::unique_ptr ptr; -}; - -} // detail -} // toml -#endif// TOML11_STORAGE_HPP diff -Nru nix-2.8.0/src/toml11/toml/string.hpp nix-2.16.1+dfsg/src/toml11/toml/string.hpp --- nix-2.8.0/src/toml11/toml/string.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/string.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,225 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_STRING_HPP -#define TOML11_STRING_HPP -#include - -#include -#include - -#if __cplusplus >= 201703L -#if __has_include() -#define TOML11_USING_STRING_VIEW 1 -#include -#endif -#endif - -namespace toml -{ - -enum class string_t : std::uint8_t -{ - basic = 0, - literal = 1, -}; - -struct string -{ - string() = default; - ~string() = default; - string(const string& s) = default; - string(string&& s) = default; - string& operator=(const string& s) = default; - string& operator=(string&& s) = default; - - string(const std::string& s): kind(string_t::basic), str(s){} - string(const std::string& s, string_t k): kind(k), str(s){} - string(const char* s): kind(string_t::basic), str(s){} - string(const char* s, string_t k): kind(k), str(s){} - - string(std::string&& s): kind(string_t::basic), str(std::move(s)){} - string(std::string&& s, string_t k): kind(k), str(std::move(s)){} - - string& operator=(const std::string& s) - {kind = string_t::basic; str = s; return *this;} - string& operator=(std::string&& s) - {kind = string_t::basic; str = std::move(s); return *this;} - - operator std::string& () & noexcept {return str;} - operator std::string const& () const& noexcept {return str;} - operator std::string&& () && noexcept {return std::move(str);} - - string& operator+=(const char* rhs) {str += rhs; return *this;} - string& operator+=(const char rhs) {str += rhs; return *this;} - string& operator+=(const std::string& rhs) {str += rhs; return *this;} - string& operator+=(const string& rhs) {str += rhs.str; return *this;} - -#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 - explicit string(std::string_view s): kind(string_t::basic), str(s){} - string(std::string_view s, string_t k): kind(k), str(s){} - - string& operator=(std::string_view s) - {kind = string_t::basic; str = s; return *this;} - - explicit operator std::string_view() const noexcept - {return std::string_view(str);} - - string& operator+=(const std::string_view& rhs) {str += rhs; return *this;} -#endif - - string_t kind; - std::string str; -}; - -inline bool operator==(const string& lhs, const string& rhs) -{ - return lhs.kind == rhs.kind && lhs.str == rhs.str; -} -inline bool operator!=(const string& lhs, const string& rhs) -{ - return !(lhs == rhs); -} -inline bool operator<(const string& lhs, const string& rhs) -{ - return (lhs.kind == rhs.kind) ? (lhs.str < rhs.str) : (lhs.kind < rhs.kind); -} -inline bool operator>(const string& lhs, const string& rhs) -{ - return rhs < lhs; -} -inline bool operator<=(const string& lhs, const string& rhs) -{ - return !(rhs < lhs); -} -inline bool operator>=(const string& lhs, const string& rhs) -{ - return !(lhs < rhs); -} - -inline bool -operator==(const string& lhs, const std::string& rhs) {return lhs.str == rhs;} -inline bool -operator!=(const string& lhs, const std::string& rhs) {return lhs.str != rhs;} -inline bool -operator< (const string& lhs, const std::string& rhs) {return lhs.str < rhs;} -inline bool -operator> (const string& lhs, const std::string& rhs) {return lhs.str > rhs;} -inline bool -operator<=(const string& lhs, const std::string& rhs) {return lhs.str <= rhs;} -inline bool -operator>=(const string& lhs, const std::string& rhs) {return lhs.str >= rhs;} - -inline bool -operator==(const std::string& lhs, const string& rhs) {return lhs == rhs.str;} -inline bool -operator!=(const std::string& lhs, const string& rhs) {return lhs != rhs.str;} -inline bool -operator< (const std::string& lhs, const string& rhs) {return lhs < rhs.str;} -inline bool -operator> (const std::string& lhs, const string& rhs) {return lhs > rhs.str;} -inline bool -operator<=(const std::string& lhs, const string& rhs) {return lhs <= rhs.str;} -inline bool -operator>=(const std::string& lhs, const string& rhs) {return lhs >= rhs.str;} - -inline bool -operator==(const string& lhs, const char* rhs) {return lhs.str == std::string(rhs);} -inline bool -operator!=(const string& lhs, const char* rhs) {return lhs.str != std::string(rhs);} -inline bool -operator< (const string& lhs, const char* rhs) {return lhs.str < std::string(rhs);} -inline bool -operator> (const string& lhs, const char* rhs) {return lhs.str > std::string(rhs);} -inline bool -operator<=(const string& lhs, const char* rhs) {return lhs.str <= std::string(rhs);} -inline bool -operator>=(const string& lhs, const char* rhs) {return lhs.str >= std::string(rhs);} - -inline bool -operator==(const char* lhs, const string& rhs) {return std::string(lhs) == rhs.str;} -inline bool -operator!=(const char* lhs, const string& rhs) {return std::string(lhs) != rhs.str;} -inline bool -operator< (const char* lhs, const string& rhs) {return std::string(lhs) < rhs.str;} -inline bool -operator> (const char* lhs, const string& rhs) {return std::string(lhs) > rhs.str;} -inline bool -operator<=(const char* lhs, const string& rhs) {return std::string(lhs) <= rhs.str;} -inline bool -operator>=(const char* lhs, const string& rhs) {return std::string(lhs) >= rhs.str;} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const string& s) -{ - if(s.kind == string_t::basic) - { - if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend()) - { - // it contains newline. make it multiline string. - os << "\"\"\"\n"; - for(auto i=s.str.cbegin(), e=s.str.cend(); i!=e; ++i) - { - switch(*i) - { - case '\\': {os << "\\\\"; break;} - case '\"': {os << "\\\""; break;} - case '\b': {os << "\\b"; break;} - case '\t': {os << "\\t"; break;} - case '\f': {os << "\\f"; break;} - case '\n': {os << '\n'; break;} - case '\r': - { - // since it is a multiline string, - // CRLF is not needed to be escaped. - if(std::next(i) != e && *std::next(i) == '\n') - { - os << "\r\n"; - ++i; - } - else - { - os << "\\r"; - } - break; - } - default: {os << *i; break;} - } - } - os << "\\\n\"\"\""; - return os; - } - // no newline. make it inline. - os << "\""; - for(const auto c : s.str) - { - switch(c) - { - case '\\': {os << "\\\\"; break;} - case '\"': {os << "\\\""; break;} - case '\b': {os << "\\b"; break;} - case '\t': {os << "\\t"; break;} - case '\f': {os << "\\f"; break;} - case '\n': {os << "\\n"; break;} - case '\r': {os << "\\r"; break;} - default : {os << c; break;} - } - } - os << "\""; - return os; - } - // the string `s` is literal-string. - if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || - std::find(s.str.cbegin(), s.str.cend(), '\'') != s.str.cend() ) - { - // contains newline or single quote. make it multiline. - os << "'''\n" << s.str << "'''"; - return os; - } - // normal literal string - os << '\'' << s.str << '\''; - return os; -} - -} // toml -#endif// TOML11_STRING_H diff -Nru nix-2.8.0/src/toml11/toml/traits.hpp nix-2.16.1+dfsg/src/toml11/toml/traits.hpp --- nix-2.8.0/src/toml11/toml/traits.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/traits.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,327 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_TRAITS_HPP -#define TOML11_TRAITS_HPP - -#include "from.hpp" -#include "into.hpp" - -#include -#include -#include -#include -#include -#include - -#if __cplusplus >= 201703L -#if __has_include() -#include -#endif // has_include() -#endif // cplusplus >= C++17 - -namespace toml -{ -template class T, template class A> -class basic_value; - -namespace detail -{ -// --------------------------------------------------------------------------- -// check whether type T is a kind of container/map class - -struct has_iterator_impl -{ - template static std::true_type check(typename T::iterator*); - template static std::false_type check(...); -}; -struct has_value_type_impl -{ - template static std::true_type check(typename T::value_type*); - template static std::false_type check(...); -}; -struct has_key_type_impl -{ - template static std::true_type check(typename T::key_type*); - template static std::false_type check(...); -}; -struct has_mapped_type_impl -{ - template static std::true_type check(typename T::mapped_type*); - template static std::false_type check(...); -}; -struct has_reserve_method_impl -{ - template static std::false_type check(...); - template static std::true_type check( - decltype(std::declval().reserve(std::declval()))*); -}; -struct has_push_back_method_impl -{ - template static std::false_type check(...); - template static std::true_type check( - decltype(std::declval().push_back(std::declval()))*); -}; -struct is_comparable_impl -{ - template static std::false_type check(...); - template static std::true_type check( - decltype(std::declval() < std::declval())*); -}; - -struct has_from_toml_method_impl -{ - template class Tb, template class A> - static std::true_type check( - decltype(std::declval().from_toml( - std::declval<::toml::basic_value>()))*); - - template class Tb, template class A> - static std::false_type check(...); -}; -struct has_into_toml_method_impl -{ - template - static std::true_type check(decltype(std::declval().into_toml())*); - template - static std::false_type check(...); -}; - -struct has_specialized_from_impl -{ - template - static std::false_type check(...); - template)> - static std::true_type check(::toml::from*); -}; -struct has_specialized_into_impl -{ - template - static std::false_type check(...); - template)> - static std::true_type check(::toml::from*); -}; - - -/// Intel C++ compiler can not use decltype in parent class declaration, here -/// is a hack to work around it. https://stackoverflow.com/a/23953090/4692076 -#ifdef __INTEL_COMPILER -#define decltype(...) std::enable_if::type -#endif - -template -struct has_iterator : decltype(has_iterator_impl::check(nullptr)){}; -template -struct has_value_type : decltype(has_value_type_impl::check(nullptr)){}; -template -struct has_key_type : decltype(has_key_type_impl::check(nullptr)){}; -template -struct has_mapped_type : decltype(has_mapped_type_impl::check(nullptr)){}; -template -struct has_reserve_method : decltype(has_reserve_method_impl::check(nullptr)){}; -template -struct has_push_back_method : decltype(has_push_back_method_impl::check(nullptr)){}; -template -struct is_comparable : decltype(is_comparable_impl::check(nullptr)){}; - -template class Tb, template class A> -struct has_from_toml_method -: decltype(has_from_toml_method_impl::check(nullptr)){}; - -template -struct has_into_toml_method -: decltype(has_into_toml_method_impl::check(nullptr)){}; - -template -struct has_specialized_from : decltype(has_specialized_from_impl::check(nullptr)){}; -template -struct has_specialized_into : decltype(has_specialized_into_impl::check(nullptr)){}; - -#ifdef __INTEL_COMPILER -#undef decltype -#endif - -// --------------------------------------------------------------------------- -// C++17 and/or/not - -#if __cplusplus >= 201703L - -using std::conjunction; -using std::disjunction; -using std::negation; - -#else - -template struct conjunction : std::true_type{}; -template struct conjunction : T{}; -template -struct conjunction : - std::conditional(T::value), conjunction, T>::type -{}; - -template struct disjunction : std::false_type{}; -template struct disjunction : T {}; -template -struct disjunction : - std::conditional(T::value), T, disjunction>::type -{}; - -template -struct negation : std::integral_constant(T::value)>{}; - -#endif - -// --------------------------------------------------------------------------- -// type checkers - -template struct is_std_pair : std::false_type{}; -template -struct is_std_pair> : std::true_type{}; - -template struct is_std_tuple : std::false_type{}; -template -struct is_std_tuple> : std::true_type{}; - -template struct is_std_forward_list : std::false_type{}; -template -struct is_std_forward_list> : std::true_type{}; - -template struct is_chrono_duration: std::false_type{}; -template -struct is_chrono_duration>: std::true_type{}; - -template -struct is_map : conjunction< // map satisfies all the following conditions - has_iterator, // has T::iterator - has_value_type, // has T::value_type - has_key_type, // has T::key_type - has_mapped_type // has T::mapped_type - >{}; -template struct is_map : is_map{}; -template struct is_map : is_map{}; -template struct is_map : is_map{}; -template struct is_map : is_map{}; - -template -struct is_container : conjunction< - negation>, // not a map - negation>, // not a std::string -#if __cplusplus >= 201703L -#if __has_include() - negation>, // not a std::string_view -#endif // has_include() -#endif - has_iterator, // has T::iterator - has_value_type // has T::value_type - >{}; -template struct is_container : is_container{}; -template struct is_container : is_container{}; -template struct is_container : is_container{}; -template struct is_container : is_container{}; - -template -struct is_basic_value: std::false_type{}; -template struct is_basic_value : is_basic_value{}; -template struct is_basic_value : is_basic_value{}; -template struct is_basic_value : is_basic_value{}; -template struct is_basic_value : is_basic_value{}; -template class M, template class V> -struct is_basic_value<::toml::basic_value>: std::true_type{}; - -// --------------------------------------------------------------------------- -// C++14 index_sequence - -#if __cplusplus >= 201402L - -using std::index_sequence; -using std::make_index_sequence; - -#else - -template struct index_sequence{}; - -template struct push_back_index_sequence{}; -template -struct push_back_index_sequence, N> -{ - typedef index_sequence type; -}; - -template -struct index_sequence_maker -{ - typedef typename push_back_index_sequence< - typename index_sequence_maker::type, N>::type type; -}; -template<> -struct index_sequence_maker<0> -{ - typedef index_sequence<0> type; -}; -template -using make_index_sequence = typename index_sequence_maker::type; - -#endif // __cplusplus >= 2014 - -// --------------------------------------------------------------------------- -// C++14 enable_if_t - -#if __cplusplus >= 201402L - -using std::enable_if_t; - -#else - -template -using enable_if_t = typename std::enable_if::type; - -#endif // __cplusplus >= 2014 - -// --------------------------------------------------------------------------- -// return_type_of_t - -#if __cplusplus >= 201703L && defined(__cpp_lib_is_invocable) && __cpp_lib_is_invocable>=201703 - -template -using return_type_of_t = std::invoke_result_t; - -#else -// result_of is deprecated after C++17 -template -using return_type_of_t = typename std::result_of::type; - -#endif - -// --------------------------------------------------------------------------- -// is_string_literal -// -// to use this, pass `typename remove_reference::type` to T. - -template -struct is_string_literal: -disjunction< - std::is_same, - conjunction< - std::is_array, - std::is_same::type> - > - >{}; - -// --------------------------------------------------------------------------- -// C++20 remove_cvref_t - -template -struct remove_cvref -{ - using type = typename std::remove_cv< - typename std::remove_reference::type>::type; -}; - -template -using remove_cvref_t = typename remove_cvref::type; - -}// detail -}//toml -#endif // TOML_TRAITS diff -Nru nix-2.8.0/src/toml11/toml/types.hpp nix-2.16.1+dfsg/src/toml11/toml/types.hpp --- nix-2.8.0/src/toml11/toml/types.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/types.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,173 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_TYPES_HPP -#define TOML11_TYPES_HPP -#include -#include - -#include "comments.hpp" -#include "datetime.hpp" -#include "string.hpp" -#include "traits.hpp" - -namespace toml -{ - -template class Table, // map-like class - template class Array> // vector-like class -class basic_value; - -using character = char; -using key = std::string; - -#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ <= 4 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wshadow" -#endif - -using boolean = bool; -using integer = std::int64_t; -using floating = double; // "float" is a keyword, cannot use it here. -// the following stuffs are structs defined here, so aliases are not needed. -// - string -// - offset_datetime -// - offset_datetime -// - local_datetime -// - local_date -// - local_time - -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic pop -#endif - -// default toml::value and default array/table. these are defined after defining -// basic_value itself. -// using value = basic_value; -// using array = typename value::array_type; -// using table = typename value::table_type; - -// to avoid warnings about `value_t::integer` is "shadowing" toml::integer in -// GCC -Wshadow=global. -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic push -# if 7 <= __GNUC__ -# pragma GCC diagnostic ignored "-Wshadow=global" -# else // gcc-6 or older -# pragma GCC diagnostic ignored "-Wshadow" -# endif -#endif -enum class value_t : std::uint8_t -{ - empty = 0, - boolean = 1, - integer = 2, - floating = 3, - string = 4, - offset_datetime = 5, - local_datetime = 6, - local_date = 7, - local_time = 8, - array = 9, - table = 10, -}; -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic pop -#endif - -template -inline std::basic_ostream& -operator<<(std::basic_ostream& os, value_t t) -{ - switch(t) - { - case value_t::boolean : os << "boolean"; return os; - case value_t::integer : os << "integer"; return os; - case value_t::floating : os << "floating"; return os; - case value_t::string : os << "string"; return os; - case value_t::offset_datetime : os << "offset_datetime"; return os; - case value_t::local_datetime : os << "local_datetime"; return os; - case value_t::local_date : os << "local_date"; return os; - case value_t::local_time : os << "local_time"; return os; - case value_t::array : os << "array"; return os; - case value_t::table : os << "table"; return os; - case value_t::empty : os << "empty"; return os; - default : os << "unknown"; return os; - } -} - -template, - typename alloc = std::allocator> -inline std::basic_string stringize(value_t t) -{ - std::basic_ostringstream oss; - oss << t; - return oss.str(); -} - -namespace detail -{ - -// helper to define a type that represents a value_t value. -template -using value_t_constant = std::integral_constant; - -// meta-function that convertes from value_t to the exact toml type that corresponds to. -// It takes toml::basic_value type because array and table types depend on it. -template struct enum_to_type {using type = void ;}; -template struct enum_to_type{using type = void ;}; -template struct enum_to_type{using type = boolean ;}; -template struct enum_to_type{using type = integer ;}; -template struct enum_to_type{using type = floating ;}; -template struct enum_to_type{using type = string ;}; -template struct enum_to_type{using type = offset_datetime ;}; -template struct enum_to_type{using type = local_datetime ;}; -template struct enum_to_type{using type = local_date ;}; -template struct enum_to_type{using type = local_time ;}; -template struct enum_to_type{using type = typename Value::array_type;}; -template struct enum_to_type{using type = typename Value::table_type;}; - -// meta-function that converts from an exact toml type to the enum that corresponds to. -template -struct type_to_enum : std::conditional< - std::is_same::value, // if T == array_type, - value_t_constant, // then value_t::array - typename std::conditional< // else... - std::is_same::value, // if T == table_type - value_t_constant, // then value_t::table - value_t_constant // else value_t::empty - >::type - >::type {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; - -// meta-function that checks the type T is the same as one of the toml::* types. -template -struct is_exact_toml_type : disjunction< - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same - >{}; -template struct is_exact_toml_type : is_exact_toml_type{}; -template struct is_exact_toml_type : is_exact_toml_type{}; -template struct is_exact_toml_type : is_exact_toml_type{}; -template struct is_exact_toml_type: is_exact_toml_type{}; - -} // detail -} // toml - -#endif// TOML11_TYPES_H diff -Nru nix-2.8.0/src/toml11/toml/utility.hpp nix-2.16.1+dfsg/src/toml11/toml/utility.hpp --- nix-2.8.0/src/toml11/toml/utility.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/utility.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,149 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_UTILITY_HPP -#define TOML11_UTILITY_HPP -#include -#include -#include - -#include "traits.hpp" - -#if __cplusplus >= 201402L -# define TOML11_MARK_AS_DEPRECATED(msg) [[deprecated(msg)]] -#elif defined(__GNUC__) -# define TOML11_MARK_AS_DEPRECATED(msg) __attribute__((deprecated(msg))) -#elif defined(_MSC_VER) -# define TOML11_MARK_AS_DEPRECATED(msg) __declspec(deprecated(msg)) -#else -# define TOML11_MARK_AS_DEPRECATED -#endif - -namespace toml -{ - -#if __cplusplus >= 201402L - -using std::make_unique; - -#else - -template -inline std::unique_ptr make_unique(Ts&& ... args) -{ - return std::unique_ptr(new T(std::forward(args)...)); -} - -#endif // __cplusplus >= 2014 - -namespace detail -{ -template -void try_reserve_impl(Container& container, std::size_t N, std::true_type) -{ - container.reserve(N); - return; -} -template -void try_reserve_impl(Container&, std::size_t, std::false_type) noexcept -{ - return; -} -} // detail - -template -void try_reserve(Container& container, std::size_t N) -{ - if(N <= container.size()) {return;} - detail::try_reserve_impl(container, N, detail::has_reserve_method{}); - return; -} - -namespace detail -{ -inline std::string concat_to_string_impl(std::ostringstream& oss) -{ - return oss.str(); -} -template -std::string concat_to_string_impl(std::ostringstream& oss, T&& head, Ts&& ... tail) -{ - oss << std::forward(head); - return concat_to_string_impl(oss, std::forward(tail) ... ); -} -} // detail - -template -std::string concat_to_string(Ts&& ... args) -{ - std::ostringstream oss; - oss << std::boolalpha << std::fixed; - return detail::concat_to_string_impl(oss, std::forward(args) ...); -} - -template -T from_string(const std::string& str, T opt) -{ - T v(opt); - std::istringstream iss(str); - iss >> v; - return v; -} - -namespace detail -{ -#if __cplusplus >= 201402L -template -decltype(auto) last_one(T&& tail) noexcept -{ - return std::forward(tail); -} - -template -decltype(auto) last_one(T&& /*head*/, Ts&& ... tail) noexcept -{ - return last_one(std::forward(tail)...); -} -#else // C++11 -// The following code -// ```cpp -// 1 | template -// 2 | auto last_one(T&& /*head*/, Ts&& ... tail) -// 3 | -> decltype(last_one(std::forward(tail)...)) -// 4 | { -// 5 | return last_one(std::forward(tail)...); -// 6 | } -// ``` -// does not work because the function `last_one(...)` is not yet defined at -// line #3, so `decltype()` cannot deduce the type returned from `last_one`. -// So we need to determine return type in a different way, like a meta func. - -template -struct last_one_in_pack -{ - using type = typename last_one_in_pack::type; -}; -template -struct last_one_in_pack -{ - using type = T; -}; -template -using last_one_in_pack_t = typename last_one_in_pack::type; - -template -T&& last_one(T&& tail) noexcept -{ - return std::forward(tail); -} -template -enable_if_t<(sizeof...(Ts) > 0), last_one_in_pack_t> -last_one(T&& /*head*/, Ts&& ... tail) -{ - return last_one(std::forward(tail)...); -} - -#endif -} // detail - -}// toml -#endif // TOML11_UTILITY diff -Nru nix-2.8.0/src/toml11/toml/value.hpp nix-2.16.1+dfsg/src/toml11/toml/value.hpp --- nix-2.8.0/src/toml11/toml/value.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml/value.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,2035 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_VALUE_HPP -#define TOML11_VALUE_HPP -#include - -#include "comments.hpp" -#include "exception.hpp" -#include "into.hpp" -#include "region.hpp" -#include "source_location.hpp" -#include "storage.hpp" -#include "traits.hpp" -#include "types.hpp" -#include "utility.hpp" - -namespace toml -{ - -namespace detail -{ - -// to show error messages. not recommended for users. -template -inline region_base const* get_region(const Value& v) -{ - return v.region_info_.get(); -} - -template -void change_region(Value& v, region reg) -{ - v.region_info_ = std::make_shared(std::move(reg)); - return; -} - -template -[[noreturn]] inline void -throw_bad_cast(const std::string& funcname, value_t actual, const Value& v) -{ - throw type_error(detail::format_underline( - concat_to_string(funcname, "bad_cast to ", Expected), { - {v.location(), concat_to_string("the actual type is ", actual)} - }), v.location()); -} - -// Throw `out_of_range` from `toml::value::at()` and `toml::find()` -// after generating an error message. -// -// The implementation is a bit complicated and there are many edge-cases. -// If you are not interested in the error message generation, just skip this. -template -[[noreturn]] void -throw_key_not_found_error(const Value& v, const key& ky) -{ - // The top-level table has its region at the first character of the file. - // That means that, in the case when a key is not found in the top-level - // table, the error message points to the first character. If the file has - // its first table at the first line, the error message would be like this. - // ```console - // [error] key "a" not found - // --> example.toml - // | - // 1 | [table] - // | ^------ in this table - // ``` - // It actually points to the top-level table at the first character, - // not `[table]`. But it is too confusing. To avoid the confusion, the error - // message should explicitly say "key not found in the top-level table", - // or "the parsed file is empty" if there is no content at all (0 bytes in file). - const auto loc = v.location(); - if(loc.line() == 1 && loc.region() == 0) - { - // First line with a zero-length region means "empty file". - // The region will be generated at `parse_toml_file` function - // if the file contains no bytes. - throw std::out_of_range(format_underline(concat_to_string( - "key \"", ky, "\" not found in the top-level table"), { - {loc, "the parsed file is empty"} - })); - } - else if(loc.line() == 1 && loc.region() == 1) - { - // Here it assumes that top-level table starts at the first character. - // The region corresponds to the top-level table will be generated at - // `parse_toml_file` function. - // It also assumes that the top-level table size is just one and - // the line number is `1`. It is always satisfied. And those conditions - // are satisfied only if the table is the top-level table. - // - // 1. one-character dot-key at the first line - // ```toml - // a.b = "c" - // ``` - // toml11 counts whole key as the table key. Here, `a.b` is the region - // of the table "a". It could be counter intuitive, but it works. - // The size of the region is 3, not 1. The above example is the shortest - // dot-key example. The size cannot be 1. - // - // 2. one-character inline-table at the first line - // ```toml - // a = {b = "c"} - // ``` - // toml11 considers the inline table body as the table region. Here, - // `{b = "c"}` is the region of the table "a". The size of the region - // is 9, not 1. The shotest inline table still has two characters, `{` - // and `}`. The size cannot be 1. - // - // 3. one-character table declaration at the first line - // ```toml - // [a] - // ``` - // toml11 considers the whole table key as the table region. Here, - // `[a]` is the table region. The size is 3, not 1. - // - throw std::out_of_range(format_underline(concat_to_string( - "key \"", ky, "\" not found in the top-level table"), { - {loc, "the top-level table starts here"} - })); - } - else - { - // normal table. - throw std::out_of_range(format_underline(concat_to_string( - "key \"", ky, "\" not found"), { {loc, "in this table"} })); - } -} - -// switch by `value_t` at the compile time. -template -struct switch_cast {}; -#define TOML11_GENERATE_SWITCH_CASTER(TYPE) \ - template<> \ - struct switch_cast \ - { \ - template \ - static typename Value::TYPE##_type& invoke(Value& v) \ - { \ - return v.as_##TYPE(); \ - } \ - template \ - static typename Value::TYPE##_type const& invoke(const Value& v) \ - { \ - return v.as_##TYPE(); \ - } \ - template \ - static typename Value::TYPE##_type&& invoke(Value&& v) \ - { \ - return std::move(v).as_##TYPE(); \ - } \ - }; \ - /**/ -TOML11_GENERATE_SWITCH_CASTER(boolean) -TOML11_GENERATE_SWITCH_CASTER(integer) -TOML11_GENERATE_SWITCH_CASTER(floating) -TOML11_GENERATE_SWITCH_CASTER(string) -TOML11_GENERATE_SWITCH_CASTER(offset_datetime) -TOML11_GENERATE_SWITCH_CASTER(local_datetime) -TOML11_GENERATE_SWITCH_CASTER(local_date) -TOML11_GENERATE_SWITCH_CASTER(local_time) -TOML11_GENERATE_SWITCH_CASTER(array) -TOML11_GENERATE_SWITCH_CASTER(table) - -#undef TOML11_GENERATE_SWITCH_CASTER - -}// detail - -template class Table = std::unordered_map, - template class Array = std::vector> -class basic_value -{ - template - static void assigner(T& dst, U&& v) - { - const auto tmp = ::new(std::addressof(dst)) T(std::forward(v)); - assert(tmp == std::addressof(dst)); - (void)tmp; - } - - using region_base = detail::region_base; - - template class T, - template class A> - friend class basic_value; - - public: - - using comment_type = Comment; - using key_type = ::toml::key; - using value_type = basic_value; - using boolean_type = ::toml::boolean; - using integer_type = ::toml::integer; - using floating_type = ::toml::floating; - using string_type = ::toml::string; - using local_time_type = ::toml::local_time; - using local_date_type = ::toml::local_date; - using local_datetime_type = ::toml::local_datetime; - using offset_datetime_type = ::toml::offset_datetime; - using array_type = Array; - using table_type = Table; - - public: - - basic_value() noexcept - : type_(value_t::empty), - region_info_(std::make_shared(region_base{})) - {} - ~basic_value() noexcept {this->cleanup();} - - basic_value(const basic_value& v) - : type_(v.type()), region_info_(v.region_info_), comments_(v.comments_) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : assigner(array_ , v.array_ ); break; - case value_t::table : assigner(table_ , v.table_ ); break; - default: break; - } - } - basic_value(basic_value&& v) - : type_(v.type()), region_info_(std::move(v.region_info_)), - comments_(std::move(v.comments_)) - { - switch(this->type_) // here this->type_ is already initialized - { - case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; - case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; - case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; - case value_t::string : assigner(string_ , std::move(v.string_ )); break; - case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; - case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; - case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; - case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; - case value_t::array : assigner(array_ , std::move(v.array_ )); break; - case value_t::table : assigner(table_ , std::move(v.table_ )); break; - default: break; - } - } - basic_value& operator=(const basic_value& v) - { - this->cleanup(); - this->region_info_ = v.region_info_; - this->comments_ = v.comments_; - this->type_ = v.type(); - switch(this->type_) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : assigner(array_ , v.array_ ); break; - case value_t::table : assigner(table_ , v.table_ ); break; - default: break; - } - return *this; - } - basic_value& operator=(basic_value&& v) - { - this->cleanup(); - this->region_info_ = std::move(v.region_info_); - this->comments_ = std::move(v.comments_); - this->type_ = v.type(); - switch(this->type_) - { - case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; - case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; - case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; - case value_t::string : assigner(string_ , std::move(v.string_ )); break; - case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; - case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; - case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; - case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; - case value_t::array : assigner(array_ , std::move(v.array_ )); break; - case value_t::table : assigner(table_ , std::move(v.table_ )); break; - default: break; - } - return *this; - } - - // overwrite comments ---------------------------------------------------- - - basic_value(const basic_value& v, std::vector com) - : type_(v.type()), region_info_(v.region_info_), - comments_(std::move(com)) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : assigner(array_ , v.array_ ); break; - case value_t::table : assigner(table_ , v.table_ ); break; - default: break; - } - } - - basic_value(basic_value&& v, std::vector com) - : type_(v.type()), region_info_(std::move(v.region_info_)), - comments_(std::move(com)) - { - switch(this->type_) // here this->type_ is already initialized - { - case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; - case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; - case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; - case value_t::string : assigner(string_ , std::move(v.string_ )); break; - case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; - case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; - case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; - case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; - case value_t::array : assigner(array_ , std::move(v.array_ )); break; - case value_t::table : assigner(table_ , std::move(v.table_ )); break; - default: break; - } - } - - // ----------------------------------------------------------------------- - // conversion between different basic_values. - template class T, - template class A> - basic_value(const basic_value& v) - : type_(v.type()), region_info_(v.region_info_), comments_(v.comments()) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : - { - array_type tmp(v.as_array(std::nothrow).begin(), - v.as_array(std::nothrow).end()); - assigner(array_, std::move(tmp)); - break; - } - case value_t::table : - { - table_type tmp(v.as_table(std::nothrow).begin(), - v.as_table(std::nothrow).end()); - assigner(table_, std::move(tmp)); - break; - } - default: break; - } - } - template class T, - template class A> - basic_value(const basic_value& v, std::vector com) - : type_(v.type()), region_info_(v.region_info_), - comments_(std::move(com)) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : - { - array_type tmp(v.as_array(std::nothrow).begin(), - v.as_array(std::nothrow).end()); - assigner(array_, std::move(tmp)); - break; - } - case value_t::table : - { - table_type tmp(v.as_table(std::nothrow).begin(), - v.as_table(std::nothrow).end()); - assigner(table_, std::move(tmp)); - break; - } - default: break; - } - } - template class T, - template class A> - basic_value& operator=(const basic_value& v) - { - this->region_info_ = v.region_info_; - this->comments_ = comment_type(v.comments()); - this->type_ = v.type(); - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : - { - array_type tmp(v.as_array(std::nothrow).begin(), - v.as_array(std::nothrow).end()); - assigner(array_, std::move(tmp)); - break; - } - case value_t::table : - { - table_type tmp(v.as_table(std::nothrow).begin(), - v.as_table(std::nothrow).end()); - assigner(table_, std::move(tmp)); - break; - } - default: break; - } - return *this; - } - - // boolean ============================================================== - - basic_value(boolean b) - : type_(value_t::boolean), - region_info_(std::make_shared(region_base{})) - { - assigner(this->boolean_, b); - } - basic_value& operator=(boolean b) - { - this->cleanup(); - this->type_ = value_t::boolean; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->boolean_, b); - return *this; - } - basic_value(boolean b, std::vector com) - : type_(value_t::boolean), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->boolean_, b); - } - - // integer ============================================================== - - template, detail::negation>>::value, - std::nullptr_t>::type = nullptr> - basic_value(T i) - : type_(value_t::integer), - region_info_(std::make_shared(region_base{})) - { - assigner(this->integer_, static_cast(i)); - } - - template, detail::negation>>::value, - std::nullptr_t>::type = nullptr> - basic_value& operator=(T i) - { - this->cleanup(); - this->type_ = value_t::integer; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->integer_, static_cast(i)); - return *this; - } - - template, detail::negation>>::value, - std::nullptr_t>::type = nullptr> - basic_value(T i, std::vector com) - : type_(value_t::integer), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->integer_, static_cast(i)); - } - - // floating ============================================================= - - template::value, std::nullptr_t>::type = nullptr> - basic_value(T f) - : type_(value_t::floating), - region_info_(std::make_shared(region_base{})) - { - assigner(this->floating_, static_cast(f)); - } - - - template::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(T f) - { - this->cleanup(); - this->type_ = value_t::floating; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->floating_, static_cast(f)); - return *this; - } - - template::value, std::nullptr_t>::type = nullptr> - basic_value(T f, std::vector com) - : type_(value_t::floating), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->floating_, f); - } - - // string =============================================================== - - basic_value(toml::string s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, std::move(s)); - } - basic_value& operator=(toml::string s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, s); - return *this; - } - basic_value(toml::string s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, std::move(s)); - } - - basic_value(std::string s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::move(s))); - } - basic_value& operator=(std::string s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, toml::string(std::move(s))); - return *this; - } - basic_value(std::string s, string_t kind) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::move(s), kind)); - } - basic_value(std::string s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::move(s))); - } - basic_value(std::string s, string_t kind, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::move(s), kind)); - } - - basic_value(const char* s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::string(s))); - } - basic_value& operator=(const char* s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, toml::string(std::string(s))); - return *this; - } - basic_value(const char* s, string_t kind) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::string(s), kind)); - } - basic_value(const char* s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::string(s))); - } - basic_value(const char* s, string_t kind, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::string(s), kind)); - } - -#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 - basic_value(std::string_view s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(s)); - } - basic_value& operator=(std::string_view s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, toml::string(s)); - return *this; - } - basic_value(std::string_view s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(s)); - } - basic_value(std::string_view s, string_t kind) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(s, kind)); - } - basic_value(std::string_view s, string_t kind, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(s, kind)); - } -#endif - - // local date =========================================================== - - basic_value(const local_date& ld) - : type_(value_t::local_date), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_date_, ld); - } - basic_value& operator=(const local_date& ld) - { - this->cleanup(); - this->type_ = value_t::local_date; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_date_, ld); - return *this; - } - basic_value(const local_date& ld, std::vector com) - : type_(value_t::local_date), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_date_, ld); - } - - // local time =========================================================== - - basic_value(const local_time& lt) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_time_, lt); - } - basic_value(const local_time& lt, std::vector com) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_time_, lt); - } - basic_value& operator=(const local_time& lt) - { - this->cleanup(); - this->type_ = value_t::local_time; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_time_, lt); - return *this; - } - - template - basic_value(const std::chrono::duration& dur) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_time_, local_time(dur)); - } - template - basic_value(const std::chrono::duration& dur, - std::vector com) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_time_, local_time(dur)); - } - template - basic_value& operator=(const std::chrono::duration& dur) - { - this->cleanup(); - this->type_ = value_t::local_time; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_time_, local_time(dur)); - return *this; - } - - // local datetime ======================================================= - - basic_value(const local_datetime& ldt) - : type_(value_t::local_datetime), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_datetime_, ldt); - } - basic_value(const local_datetime& ldt, std::vector com) - : type_(value_t::local_datetime), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_datetime_, ldt); - } - basic_value& operator=(const local_datetime& ldt) - { - this->cleanup(); - this->type_ = value_t::local_datetime; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_datetime_, ldt); - return *this; - } - - // offset datetime ====================================================== - - basic_value(const offset_datetime& odt) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})) - { - assigner(this->offset_datetime_, odt); - } - basic_value(const offset_datetime& odt, std::vector com) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->offset_datetime_, odt); - } - basic_value& operator=(const offset_datetime& odt) - { - this->cleanup(); - this->type_ = value_t::offset_datetime; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->offset_datetime_, odt); - return *this; - } - basic_value(const std::chrono::system_clock::time_point& tp) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})) - { - assigner(this->offset_datetime_, offset_datetime(tp)); - } - basic_value(const std::chrono::system_clock::time_point& tp, - std::vector com) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->offset_datetime_, offset_datetime(tp)); - } - basic_value& operator=(const std::chrono::system_clock::time_point& tp) - { - this->cleanup(); - this->type_ = value_t::offset_datetime; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->offset_datetime_, offset_datetime(tp)); - return *this; - } - - // array ================================================================ - - basic_value(const array_type& ary) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})) - { - assigner(this->array_, ary); - } - basic_value(const array_type& ary, std::vector com) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->array_, ary); - } - basic_value& operator=(const array_type& ary) - { - this->cleanup(); - this->type_ = value_t::array ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->array_, ary); - return *this; - } - - // array (initializer_list) ---------------------------------------------- - - template::value, - std::nullptr_t>::type = nullptr> - basic_value(std::initializer_list list) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})) - { - array_type ary(list.begin(), list.end()); - assigner(this->array_, std::move(ary)); - } - template::value, - std::nullptr_t>::type = nullptr> - basic_value(std::initializer_list list, std::vector com) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - array_type ary(list.begin(), list.end()); - assigner(this->array_, std::move(ary)); - } - template::value, - std::nullptr_t>::type = nullptr> - basic_value& operator=(std::initializer_list list) - { - this->cleanup(); - this->type_ = value_t::array; - this->region_info_ = std::make_shared(region_base{}); - - array_type ary(list.begin(), list.end()); - assigner(this->array_, std::move(ary)); - return *this; - } - - // array (STL Containers) ------------------------------------------------ - - template>, - detail::is_container - >::value, std::nullptr_t>::type = nullptr> - basic_value(const T& list) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})) - { - static_assert(std::is_convertible::value, - "elements of a container should be convertible to toml::value"); - - array_type ary(list.size()); - std::copy(list.begin(), list.end(), ary.begin()); - assigner(this->array_, std::move(ary)); - } - template>, - detail::is_container - >::value, std::nullptr_t>::type = nullptr> - basic_value(const T& list, std::vector com) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - static_assert(std::is_convertible::value, - "elements of a container should be convertible to toml::value"); - - array_type ary(list.size()); - std::copy(list.begin(), list.end(), ary.begin()); - assigner(this->array_, std::move(ary)); - } - template>, - detail::is_container - >::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(const T& list) - { - static_assert(std::is_convertible::value, - "elements of a container should be convertible to toml::value"); - - this->cleanup(); - this->type_ = value_t::array; - this->region_info_ = std::make_shared(region_base{}); - - array_type ary(list.size()); - std::copy(list.begin(), list.end(), ary.begin()); - assigner(this->array_, std::move(ary)); - return *this; - } - - // table ================================================================ - - basic_value(const table_type& tab) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})) - { - assigner(this->table_, tab); - } - basic_value(const table_type& tab, std::vector com) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->table_, tab); - } - basic_value& operator=(const table_type& tab) - { - this->cleanup(); - this->type_ = value_t::table; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->table_, tab); - return *this; - } - - // initializer-list ------------------------------------------------------ - - basic_value(std::initializer_list> list) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})) - { - table_type tab; - for(const auto& elem : list) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - - basic_value(std::initializer_list> list, - std::vector com) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - table_type tab; - for(const auto& elem : list) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - basic_value& operator=(std::initializer_list> list) - { - this->cleanup(); - this->type_ = value_t::table; - this->region_info_ = std::make_shared(region_base{}); - - table_type tab; - for(const auto& elem : list) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - return *this; - } - - // other table-like ----------------------------------------------------- - - template>, - detail::is_map - >::value, std::nullptr_t>::type = nullptr> - basic_value(const Map& mp) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})) - { - table_type tab; - for(const auto& elem : mp) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - template>, - detail::is_map - >::value, std::nullptr_t>::type = nullptr> - basic_value(const Map& mp, std::vector com) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - table_type tab; - for(const auto& elem : mp) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - template>, - detail::is_map - >::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(const Map& mp) - { - this->cleanup(); - this->type_ = value_t::table; - this->region_info_ = std::make_shared(region_base{}); - - table_type tab; - for(const auto& elem : mp) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - return *this; - } - - // user-defined ========================================================= - - // convert using into_toml() method ------------------------------------- - - template::value, std::nullptr_t>::type = nullptr> - basic_value(const T& ud): basic_value(ud.into_toml()) {} - - template::value, std::nullptr_t>::type = nullptr> - basic_value(const T& ud, std::vector com) - : basic_value(ud.into_toml(), std::move(com)) - {} - template::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(const T& ud) - { - *this = ud.into_toml(); - return *this; - } - - // convert using into struct ----------------------------------------- - - template)> - basic_value(const T& ud): basic_value(::toml::into::into_toml(ud)) {} - template)> - basic_value(const T& ud, std::vector com) - : basic_value(::toml::into::into_toml(ud), std::move(com)) - {} - template)> - basic_value& operator=(const T& ud) - { - *this = ::toml::into::into_toml(ud); - return *this; - } - - // for internal use ------------------------------------------------------ - // - // Those constructors take detail::region that contains parse result. - - basic_value(boolean b, detail::region reg, std::vector cm) - : type_(value_t::boolean), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->boolean_, b); - } - template, detail::negation> - >::value, std::nullptr_t>::type = nullptr> - basic_value(T i, detail::region reg, std::vector cm) - : type_(value_t::integer), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->integer_, static_cast(i)); - } - template::value, std::nullptr_t>::type = nullptr> - basic_value(T f, detail::region reg, std::vector cm) - : type_(value_t::floating), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->floating_, static_cast(f)); - } - basic_value(toml::string s, detail::region reg, - std::vector cm) - : type_(value_t::string), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->string_, std::move(s)); - } - basic_value(const local_date& ld, detail::region reg, - std::vector cm) - : type_(value_t::local_date), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->local_date_, ld); - } - basic_value(const local_time& lt, detail::region reg, - std::vector cm) - : type_(value_t::local_time), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->local_time_, lt); - } - basic_value(const local_datetime& ldt, detail::region reg, - std::vector cm) - : type_(value_t::local_datetime), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->local_datetime_, ldt); - } - basic_value(const offset_datetime& odt, detail::region reg, - std::vector cm) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->offset_datetime_, odt); - } - basic_value(const array_type& ary, detail::region reg, - std::vector cm) - : type_(value_t::array), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->array_, ary); - } - basic_value(const table_type& tab, detail::region reg, - std::vector cm) - : type_(value_t::table), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->table_, tab); - } - - template::value, - std::nullptr_t>::type = nullptr> - basic_value(std::pair parse_result, std::vector com) - : basic_value(std::move(parse_result.first), - std::move(parse_result.second), - std::move(com)) - {} - - // type checking and casting ============================================ - - template::value, - std::nullptr_t>::type = nullptr> - bool is() const noexcept - { - return detail::type_to_enum::value == this->type_; - } - bool is(value_t t) const noexcept {return t == this->type_;} - - bool is_uninitialized() const noexcept {return this->is(value_t::empty );} - bool is_boolean() const noexcept {return this->is(value_t::boolean );} - bool is_integer() const noexcept {return this->is(value_t::integer );} - bool is_floating() const noexcept {return this->is(value_t::floating );} - bool is_string() const noexcept {return this->is(value_t::string );} - bool is_offset_datetime() const noexcept {return this->is(value_t::offset_datetime);} - bool is_local_datetime() const noexcept {return this->is(value_t::local_datetime );} - bool is_local_date() const noexcept {return this->is(value_t::local_date );} - bool is_local_time() const noexcept {return this->is(value_t::local_time );} - bool is_array() const noexcept {return this->is(value_t::array );} - bool is_table() const noexcept {return this->is(value_t::table );} - - value_t type() const noexcept {return type_;} - - template - typename detail::enum_to_type::type& cast() & - { - if(this->type_ != T) - { - detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); - } - return detail::switch_cast::invoke(*this); - } - template - typename detail::enum_to_type::type const& cast() const& - { - if(this->type_ != T) - { - detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); - } - return detail::switch_cast::invoke(*this); - } - template - typename detail::enum_to_type::type&& cast() && - { - if(this->type_ != T) - { - detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); - } - return detail::switch_cast::invoke(std::move(*this)); - } - - // ------------------------------------------------------------------------ - // nothrow version - - boolean const& as_boolean (const std::nothrow_t&) const& noexcept {return this->boolean_;} - integer const& as_integer (const std::nothrow_t&) const& noexcept {return this->integer_;} - floating const& as_floating (const std::nothrow_t&) const& noexcept {return this->floating_;} - string const& as_string (const std::nothrow_t&) const& noexcept {return this->string_;} - offset_datetime const& as_offset_datetime(const std::nothrow_t&) const& noexcept {return this->offset_datetime_;} - local_datetime const& as_local_datetime (const std::nothrow_t&) const& noexcept {return this->local_datetime_;} - local_date const& as_local_date (const std::nothrow_t&) const& noexcept {return this->local_date_;} - local_time const& as_local_time (const std::nothrow_t&) const& noexcept {return this->local_time_;} - array_type const& as_array (const std::nothrow_t&) const& noexcept {return this->array_.value();} - table_type const& as_table (const std::nothrow_t&) const& noexcept {return this->table_.value();} - - boolean & as_boolean (const std::nothrow_t&) & noexcept {return this->boolean_;} - integer & as_integer (const std::nothrow_t&) & noexcept {return this->integer_;} - floating & as_floating (const std::nothrow_t&) & noexcept {return this->floating_;} - string & as_string (const std::nothrow_t&) & noexcept {return this->string_;} - offset_datetime& as_offset_datetime(const std::nothrow_t&) & noexcept {return this->offset_datetime_;} - local_datetime & as_local_datetime (const std::nothrow_t&) & noexcept {return this->local_datetime_;} - local_date & as_local_date (const std::nothrow_t&) & noexcept {return this->local_date_;} - local_time & as_local_time (const std::nothrow_t&) & noexcept {return this->local_time_;} - array_type & as_array (const std::nothrow_t&) & noexcept {return this->array_.value();} - table_type & as_table (const std::nothrow_t&) & noexcept {return this->table_.value();} - - boolean && as_boolean (const std::nothrow_t&) && noexcept {return std::move(this->boolean_);} - integer && as_integer (const std::nothrow_t&) && noexcept {return std::move(this->integer_);} - floating && as_floating (const std::nothrow_t&) && noexcept {return std::move(this->floating_);} - string && as_string (const std::nothrow_t&) && noexcept {return std::move(this->string_);} - offset_datetime&& as_offset_datetime(const std::nothrow_t&) && noexcept {return std::move(this->offset_datetime_);} - local_datetime && as_local_datetime (const std::nothrow_t&) && noexcept {return std::move(this->local_datetime_);} - local_date && as_local_date (const std::nothrow_t&) && noexcept {return std::move(this->local_date_);} - local_time && as_local_time (const std::nothrow_t&) && noexcept {return std::move(this->local_time_);} - array_type && as_array (const std::nothrow_t&) && noexcept {return std::move(this->array_.value());} - table_type && as_table (const std::nothrow_t&) && noexcept {return std::move(this->table_.value());} - - // ======================================================================== - // throw version - // ------------------------------------------------------------------------ - // const reference {{{ - - boolean const& as_boolean() const& - { - if(this->type_ != value_t::boolean) - { - detail::throw_bad_cast( - "toml::value::as_boolean(): ", this->type_, *this); - } - return this->boolean_; - } - integer const& as_integer() const& - { - if(this->type_ != value_t::integer) - { - detail::throw_bad_cast( - "toml::value::as_integer(): ", this->type_, *this); - } - return this->integer_; - } - floating const& as_floating() const& - { - if(this->type_ != value_t::floating) - { - detail::throw_bad_cast( - "toml::value::as_floating(): ", this->type_, *this); - } - return this->floating_; - } - string const& as_string() const& - { - if(this->type_ != value_t::string) - { - detail::throw_bad_cast( - "toml::value::as_string(): ", this->type_, *this); - } - return this->string_; - } - offset_datetime const& as_offset_datetime() const& - { - if(this->type_ != value_t::offset_datetime) - { - detail::throw_bad_cast( - "toml::value::as_offset_datetime(): ", this->type_, *this); - } - return this->offset_datetime_; - } - local_datetime const& as_local_datetime() const& - { - if(this->type_ != value_t::local_datetime) - { - detail::throw_bad_cast( - "toml::value::as_local_datetime(): ", this->type_, *this); - } - return this->local_datetime_; - } - local_date const& as_local_date() const& - { - if(this->type_ != value_t::local_date) - { - detail::throw_bad_cast( - "toml::value::as_local_date(): ", this->type_, *this); - } - return this->local_date_; - } - local_time const& as_local_time() const& - { - if(this->type_ != value_t::local_time) - { - detail::throw_bad_cast( - "toml::value::as_local_time(): ", this->type_, *this); - } - return this->local_time_; - } - array_type const& as_array() const& - { - if(this->type_ != value_t::array) - { - detail::throw_bad_cast( - "toml::value::as_array(): ", this->type_, *this); - } - return this->array_.value(); - } - table_type const& as_table() const& - { - if(this->type_ != value_t::table) - { - detail::throw_bad_cast( - "toml::value::as_table(): ", this->type_, *this); - } - return this->table_.value(); - } - // }}} - // ------------------------------------------------------------------------ - // nonconst reference {{{ - - boolean & as_boolean() & - { - if(this->type_ != value_t::boolean) - { - detail::throw_bad_cast( - "toml::value::as_boolean(): ", this->type_, *this); - } - return this->boolean_; - } - integer & as_integer() & - { - if(this->type_ != value_t::integer) - { - detail::throw_bad_cast( - "toml::value::as_integer(): ", this->type_, *this); - } - return this->integer_; - } - floating & as_floating() & - { - if(this->type_ != value_t::floating) - { - detail::throw_bad_cast( - "toml::value::as_floating(): ", this->type_, *this); - } - return this->floating_; - } - string & as_string() & - { - if(this->type_ != value_t::string) - { - detail::throw_bad_cast( - "toml::value::as_string(): ", this->type_, *this); - } - return this->string_; - } - offset_datetime & as_offset_datetime() & - { - if(this->type_ != value_t::offset_datetime) - { - detail::throw_bad_cast( - "toml::value::as_offset_datetime(): ", this->type_, *this); - } - return this->offset_datetime_; - } - local_datetime & as_local_datetime() & - { - if(this->type_ != value_t::local_datetime) - { - detail::throw_bad_cast( - "toml::value::as_local_datetime(): ", this->type_, *this); - } - return this->local_datetime_; - } - local_date & as_local_date() & - { - if(this->type_ != value_t::local_date) - { - detail::throw_bad_cast( - "toml::value::as_local_date(): ", this->type_, *this); - } - return this->local_date_; - } - local_time & as_local_time() & - { - if(this->type_ != value_t::local_time) - { - detail::throw_bad_cast( - "toml::value::as_local_time(): ", this->type_, *this); - } - return this->local_time_; - } - array_type & as_array() & - { - if(this->type_ != value_t::array) - { - detail::throw_bad_cast( - "toml::value::as_array(): ", this->type_, *this); - } - return this->array_.value(); - } - table_type & as_table() & - { - if(this->type_ != value_t::table) - { - detail::throw_bad_cast( - "toml::value::as_table(): ", this->type_, *this); - } - return this->table_.value(); - } - - // }}} - // ------------------------------------------------------------------------ - // rvalue reference {{{ - - boolean && as_boolean() && - { - if(this->type_ != value_t::boolean) - { - detail::throw_bad_cast( - "toml::value::as_boolean(): ", this->type_, *this); - } - return std::move(this->boolean_); - } - integer && as_integer() && - { - if(this->type_ != value_t::integer) - { - detail::throw_bad_cast( - "toml::value::as_integer(): ", this->type_, *this); - } - return std::move(this->integer_); - } - floating && as_floating() && - { - if(this->type_ != value_t::floating) - { - detail::throw_bad_cast( - "toml::value::as_floating(): ", this->type_, *this); - } - return std::move(this->floating_); - } - string && as_string() && - { - if(this->type_ != value_t::string) - { - detail::throw_bad_cast( - "toml::value::as_string(): ", this->type_, *this); - } - return std::move(this->string_); - } - offset_datetime && as_offset_datetime() && - { - if(this->type_ != value_t::offset_datetime) - { - detail::throw_bad_cast( - "toml::value::as_offset_datetime(): ", this->type_, *this); - } - return std::move(this->offset_datetime_); - } - local_datetime && as_local_datetime() && - { - if(this->type_ != value_t::local_datetime) - { - detail::throw_bad_cast( - "toml::value::as_local_datetime(): ", this->type_, *this); - } - return std::move(this->local_datetime_); - } - local_date && as_local_date() && - { - if(this->type_ != value_t::local_date) - { - detail::throw_bad_cast( - "toml::value::as_local_date(): ", this->type_, *this); - } - return std::move(this->local_date_); - } - local_time && as_local_time() && - { - if(this->type_ != value_t::local_time) - { - detail::throw_bad_cast( - "toml::value::as_local_time(): ", this->type_, *this); - } - return std::move(this->local_time_); - } - array_type && as_array() && - { - if(this->type_ != value_t::array) - { - detail::throw_bad_cast( - "toml::value::as_array(): ", this->type_, *this); - } - return std::move(this->array_.value()); - } - table_type && as_table() && - { - if(this->type_ != value_t::table) - { - detail::throw_bad_cast( - "toml::value::as_table(): ", this->type_, *this); - } - return std::move(this->table_.value()); - } - // }}} - - // accessors ============================================================= - // - // may throw type_error or out_of_range - // - value_type& at(const key& k) - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::at(key): ", this->type_, *this); - } - if(this->as_table(std::nothrow).count(k) == 0) - { - detail::throw_key_not_found_error(*this, k); - } - return this->as_table(std::nothrow).at(k); - } - value_type const& at(const key& k) const - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::at(key): ", this->type_, *this); - } - if(this->as_table(std::nothrow).count(k) == 0) - { - detail::throw_key_not_found_error(*this, k); - } - return this->as_table(std::nothrow).at(k); - } - value_type& operator[](const key& k) - { - if(this->is_uninitialized()) - { - *this = table_type{}; - } - else if(!this->is_table()) // initialized, but not a table - { - detail::throw_bad_cast( - "toml::value::operator[](key): ", this->type_, *this); - } - return this->as_table(std::nothrow)[k]; - } - - value_type& at(const std::size_t idx) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::at(idx): ", this->type_, *this); - } - if(this->as_array(std::nothrow).size() <= idx) - { - throw std::out_of_range(detail::format_underline( - "toml::value::at(idx): no element corresponding to the index", { - {this->location(), concat_to_string("the length is ", - this->as_array(std::nothrow).size(), - ", and the specified index is ", idx)} - })); - } - return this->as_array().at(idx); - } - value_type const& at(const std::size_t idx) const - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::at(idx): ", this->type_, *this); - } - if(this->as_array(std::nothrow).size() <= idx) - { - throw std::out_of_range(detail::format_underline( - "toml::value::at(idx): no element corresponding to the index", { - {this->location(), concat_to_string("the length is ", - this->as_array(std::nothrow).size(), - ", and the specified index is ", idx)} - })); - } - return this->as_array(std::nothrow).at(idx); - } - - value_type& operator[](const std::size_t idx) noexcept - { - // no check... - return this->as_array(std::nothrow)[idx]; - } - value_type const& operator[](const std::size_t idx) const noexcept - { - // no check... - return this->as_array(std::nothrow)[idx]; - } - - void push_back(const value_type& x) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::push_back(value): ", this->type_, *this); - } - this->as_array(std::nothrow).push_back(x); - return; - } - void push_back(value_type&& x) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::push_back(value): ", this->type_, *this); - } - this->as_array(std::nothrow).push_back(std::move(x)); - return; - } - - template - value_type& emplace_back(Ts&& ... args) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::emplace_back(...): ", this->type_, *this); - } - this->as_array(std::nothrow).emplace_back(std::forward(args) ...); - return this->as_array(std::nothrow).back(); - } - - std::size_t size() const - { - switch(this->type_) - { - case value_t::array: - { - return this->as_array(std::nothrow).size(); - } - case value_t::table: - { - return this->as_table(std::nothrow).size(); - } - case value_t::string: - { - return this->as_string(std::nothrow).str.size(); - } - default: - { - throw type_error(detail::format_underline( - "toml::value::size(): bad_cast to container types", { - {this->location(), - concat_to_string("the actual type is ", this->type_)} - }), this->location()); - } - } - } - - std::size_t count(const key_type& k) const - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::count(key): ", this->type_, *this); - } - return this->as_table(std::nothrow).count(k); - } - - bool contains(const key_type& k) const - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::contains(key): ", this->type_, *this); - } - return (this->as_table(std::nothrow).count(k) != 0); - } - - source_location location() const - { - return source_location(this->region_info_.get()); - } - - comment_type const& comments() const noexcept {return this->comments_;} - comment_type& comments() noexcept {return this->comments_;} - - private: - - void cleanup() noexcept - { - switch(this->type_) - { - case value_t::string : {string_.~string(); return;} - case value_t::array : {array_.~array_storage(); return;} - case value_t::table : {table_.~table_storage(); return;} - default : return; - } - } - - // for error messages - template - friend region_base const* detail::get_region(const Value& v); - - template - friend void detail::change_region(Value& v, detail::region reg); - - private: - - using array_storage = detail::storage; - using table_storage = detail::storage; - - value_t type_; - union - { - boolean boolean_; - integer integer_; - floating floating_; - string string_; - offset_datetime offset_datetime_; - local_datetime local_datetime_; - local_date local_date_; - local_time local_time_; - array_storage array_; - table_storage table_; - }; - std::shared_ptr region_info_; - comment_type comments_; -}; - -// default toml::value and default array/table. -// TOML11_DEFAULT_COMMENT_STRATEGY is defined in comments.hpp -using value = basic_value; -using array = typename value::array_type; -using table = typename value::table_type; - -template class T, template class A> -inline bool -operator==(const basic_value& lhs, const basic_value& rhs) -{ - if(lhs.type() != rhs.type()) {return false;} - if(lhs.comments() != rhs.comments()) {return false;} - - switch(lhs.type()) - { - case value_t::boolean : - { - return lhs.as_boolean() == rhs.as_boolean(); - } - case value_t::integer : - { - return lhs.as_integer() == rhs.as_integer(); - } - case value_t::floating : - { - return lhs.as_floating() == rhs.as_floating(); - } - case value_t::string : - { - return lhs.as_string() == rhs.as_string(); - } - case value_t::offset_datetime: - { - return lhs.as_offset_datetime() == rhs.as_offset_datetime(); - } - case value_t::local_datetime: - { - return lhs.as_local_datetime() == rhs.as_local_datetime(); - } - case value_t::local_date: - { - return lhs.as_local_date() == rhs.as_local_date(); - } - case value_t::local_time: - { - return lhs.as_local_time() == rhs.as_local_time(); - } - case value_t::array : - { - return lhs.as_array() == rhs.as_array(); - } - case value_t::table : - { - return lhs.as_table() == rhs.as_table(); - } - case value_t::empty : {return true; } - default: {return false;} - } -} - -template class T, template class A> -inline bool operator!=(const basic_value& lhs, const basic_value& rhs) -{ - return !(lhs == rhs); -} - -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator<(const basic_value& lhs, const basic_value& rhs) -{ - if(lhs.type() != rhs.type()){return (lhs.type() < rhs.type());} - switch(lhs.type()) - { - case value_t::boolean : - { - return lhs.as_boolean() < rhs.as_boolean() || - (lhs.as_boolean() == rhs.as_boolean() && - lhs.comments() < rhs.comments()); - } - case value_t::integer : - { - return lhs.as_integer() < rhs.as_integer() || - (lhs.as_integer() == rhs.as_integer() && - lhs.comments() < rhs.comments()); - } - case value_t::floating : - { - return lhs.as_floating() < rhs.as_floating() || - (lhs.as_floating() == rhs.as_floating() && - lhs.comments() < rhs.comments()); - } - case value_t::string : - { - return lhs.as_string() < rhs.as_string() || - (lhs.as_string() == rhs.as_string() && - lhs.comments() < rhs.comments()); - } - case value_t::offset_datetime: - { - return lhs.as_offset_datetime() < rhs.as_offset_datetime() || - (lhs.as_offset_datetime() == rhs.as_offset_datetime() && - lhs.comments() < rhs.comments()); - } - case value_t::local_datetime: - { - return lhs.as_local_datetime() < rhs.as_local_datetime() || - (lhs.as_local_datetime() == rhs.as_local_datetime() && - lhs.comments() < rhs.comments()); - } - case value_t::local_date: - { - return lhs.as_local_date() < rhs.as_local_date() || - (lhs.as_local_date() == rhs.as_local_date() && - lhs.comments() < rhs.comments()); - } - case value_t::local_time: - { - return lhs.as_local_time() < rhs.as_local_time() || - (lhs.as_local_time() == rhs.as_local_time() && - lhs.comments() < rhs.comments()); - } - case value_t::array : - { - return lhs.as_array() < rhs.as_array() || - (lhs.as_array() == rhs.as_array() && - lhs.comments() < rhs.comments()); - } - case value_t::table : - { - return lhs.as_table() < rhs.as_table() || - (lhs.as_table() == rhs.as_table() && - lhs.comments() < rhs.comments()); - } - case value_t::empty : - { - return lhs.comments() < rhs.comments(); - } - default: - { - return lhs.comments() < rhs.comments(); - } - } -} - -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator<=(const basic_value& lhs, const basic_value& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator>(const basic_value& lhs, const basic_value& rhs) -{ - return !(lhs <= rhs); -} -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator>=(const basic_value& lhs, const basic_value& rhs) -{ - return !(lhs < rhs); -} - -template class T, template class A> -inline std::string format_error(const std::string& err_msg, - const basic_value& v, const std::string& comment, - std::vector hints = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - return detail::format_underline(err_msg, {{v.location(), comment}}, - std::move(hints), colorize); -} - -template class T, template class A> -inline std::string format_error(const std::string& err_msg, - const toml::basic_value& v1, const std::string& comment1, - const toml::basic_value& v2, const std::string& comment2, - std::vector hints = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - return detail::format_underline(err_msg, { - {v1.location(), comment1}, {v2.location(), comment2} - }, std::move(hints), colorize); -} - -template class T, template class A> -inline std::string format_error(const std::string& err_msg, - const toml::basic_value& v1, const std::string& comment1, - const toml::basic_value& v2, const std::string& comment2, - const toml::basic_value& v3, const std::string& comment3, - std::vector hints = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - return detail::format_underline(err_msg, {{v1.location(), comment1}, - {v2.location(), comment2}, {v3.location(), comment3} - }, std::move(hints), colorize); -} - -template class T, template class A> -detail::return_type_of_t -visit(Visitor&& visitor, const toml::basic_value& v) -{ - switch(v.type()) - { - case value_t::boolean : {return visitor(v.as_boolean ());} - case value_t::integer : {return visitor(v.as_integer ());} - case value_t::floating : {return visitor(v.as_floating ());} - case value_t::string : {return visitor(v.as_string ());} - case value_t::offset_datetime: {return visitor(v.as_offset_datetime());} - case value_t::local_datetime : {return visitor(v.as_local_datetime ());} - case value_t::local_date : {return visitor(v.as_local_date ());} - case value_t::local_time : {return visitor(v.as_local_time ());} - case value_t::array : {return visitor(v.as_array ());} - case value_t::table : {return visitor(v.as_table ());} - case value_t::empty : break; - default: break; - } - throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " - "does not have any valid basic_value.", v, "here")); -} - -template class T, template class A> -detail::return_type_of_t -visit(Visitor&& visitor, toml::basic_value& v) -{ - switch(v.type()) - { - case value_t::boolean : {return visitor(v.as_boolean ());} - case value_t::integer : {return visitor(v.as_integer ());} - case value_t::floating : {return visitor(v.as_floating ());} - case value_t::string : {return visitor(v.as_string ());} - case value_t::offset_datetime: {return visitor(v.as_offset_datetime());} - case value_t::local_datetime : {return visitor(v.as_local_datetime ());} - case value_t::local_date : {return visitor(v.as_local_date ());} - case value_t::local_time : {return visitor(v.as_local_time ());} - case value_t::array : {return visitor(v.as_array ());} - case value_t::table : {return visitor(v.as_table ());} - case value_t::empty : break; - default: break; - } - throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " - "does not have any valid basic_value.", v, "here")); -} - -template class T, template class A> -detail::return_type_of_t -visit(Visitor&& visitor, toml::basic_value&& v) -{ - switch(v.type()) - { - case value_t::boolean : {return visitor(std::move(v.as_boolean ()));} - case value_t::integer : {return visitor(std::move(v.as_integer ()));} - case value_t::floating : {return visitor(std::move(v.as_floating ()));} - case value_t::string : {return visitor(std::move(v.as_string ()));} - case value_t::offset_datetime: {return visitor(std::move(v.as_offset_datetime()));} - case value_t::local_datetime : {return visitor(std::move(v.as_local_datetime ()));} - case value_t::local_date : {return visitor(std::move(v.as_local_date ()));} - case value_t::local_time : {return visitor(std::move(v.as_local_time ()));} - case value_t::array : {return visitor(std::move(v.as_array ()));} - case value_t::table : {return visitor(std::move(v.as_table ()));} - case value_t::empty : break; - default: break; - } - throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " - "does not have any valid basic_value.", v, "here")); -} - -}// toml -#endif// TOML11_VALUE diff -Nru nix-2.8.0/src/toml11/toml.hpp nix-2.16.1+dfsg/src/toml11/toml.hpp --- nix-2.8.0/src/toml11/toml.hpp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/src/toml11/toml.hpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2017 Toru Niina - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TOML_FOR_MODERN_CPP -#define TOML_FOR_MODERN_CPP - -#ifndef __cplusplus -# error "__cplusplus is not defined" -#endif - -#if __cplusplus < 201103L && _MSC_VER < 1900 -# error "toml11 requires C++11 or later." -#endif - -#define TOML11_VERSION_MAJOR 3 -#define TOML11_VERSION_MINOR 7 -#define TOML11_VERSION_PATCH 0 - -#include "toml/parser.hpp" -#include "toml/literal.hpp" -#include "toml/serializer.hpp" -#include "toml/get.hpp" -#include "toml/macros.hpp" - -#endif// TOML_FOR_MODERN_CPP diff -Nru nix-2.8.0/tests/binary-cache.sh nix-2.16.1+dfsg/tests/binary-cache.sh --- nix-2.8.0/tests/binary-cache.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/binary-cache.sh 2023-06-06 10:55:23.000000000 +0000 @@ -15,15 +15,15 @@ nix copy --to file://$cacheDir $outPath # Test copying build logs to the binary cache. -nix log --store file://$cacheDir $outPath 2>&1 | grep 'is not available' +expect 1 nix log --store file://$cacheDir $outPath 2>&1 | grep 'is not available' nix store copy-log --to file://$cacheDir $outPath nix log --store file://$cacheDir $outPath | grep FOO rm -rf $TEST_ROOT/var/log/nix -nix log $outPath 2>&1 | grep 'is not available' +expect 1 nix log $outPath 2>&1 | grep 'is not available' nix log --substituters file://$cacheDir $outPath | grep FOO # Test copying build logs from the binary cache. -nix store copy-log --from file://$cacheDir $(nix-store -qd $outPath) +nix store copy-log --from file://$cacheDir $(nix-store -qd $outPath)^'*' nix log $outPath | grep FOO basicDownloadTests() { @@ -78,8 +78,8 @@ mkdir -p $TEST_ROOT/empty nix-store --dump $TEST_ROOT/empty | xz > $nar -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grep -q "hash mismatch" $TEST_ROOT/log +expect 1 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log +grepQuiet "hash mismatch" $TEST_ROOT/log mv $nar.good $nar @@ -126,9 +126,9 @@ rm -v $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grep -q "copying path.*input-0" $TEST_ROOT/log -grep -q "copying path.*input-2" $TEST_ROOT/log -grep -q "copying path.*top" $TEST_ROOT/log +grepQuiet "copying path.*input-0" $TEST_ROOT/log +grepQuiet "copying path.*input-2" $TEST_ROOT/log +grepQuiet "copying path.*top" $TEST_ROOT/log # Idem, but without cached .narinfo. @@ -136,11 +136,11 @@ clearCacheCache nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grep -q "don't know how to build" $TEST_ROOT/log -grep -q "building.*input-1" $TEST_ROOT/log -grep -q "building.*input-2" $TEST_ROOT/log -grep -q "copying path.*input-0" $TEST_ROOT/log -grep -q "copying path.*top" $TEST_ROOT/log +grepQuiet "don't know how to build" $TEST_ROOT/log +grepQuiet "building.*input-1" $TEST_ROOT/log +grepQuiet "building.*input-2" $TEST_ROOT/log +grepQuiet "copying path.*input-0" $TEST_ROOT/log +grepQuiet "copying path.*top" $TEST_ROOT/log # Create a signed binary cache. diff -Nru nix-2.8.0/tests/build-delete.sh nix-2.16.1+dfsg/tests/build-delete.sh --- nix-2.8.0/tests/build-delete.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/build-delete.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,54 @@ +source common.sh + +clearStore + +# https://github.com/NixOS/nix/issues/6572 +issue_6572_independent_outputs() { + nix build -f multiple-outputs.nix --json independent --no-link > $TEST_ROOT/independent.json + + # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. + p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) + nix-store --delete "$p" # Clean up for next test + + # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. + nix-store --delete "$(jq -r <$TEST_ROOT/independent.json .[0].outputs.first)" + p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) + cmp $p < $TEST_ROOT/a.json + + # # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. + p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) + nix-store --delete "$p" # Clean up for next test + + # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. + nix-store --delete "$(jq -r <$TEST_ROOT/a.json .[0].outputs.second)" + p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) + cmp $p <&1 | grep "will be built" -# TODO: XXX: FIXME: #1793 -# Disable this part of the test until the problem is resolved: -if [ -n "$ISSUE_1795_IS_FIXED" ]; then clearStore clearCache @@ -28,7 +25,6 @@ nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" # Now old command: nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" -fi ################################################### # Check --dry-run doesn't create links with --dry-run @@ -58,7 +54,7 @@ RES=$(nix build -f dependencies.nix --dry-run --json) -if [[ -z "$NIX_TESTS_CA_BY_DEFAULT" ]]; then +if [[ -z "${NIX_TESTS_CA_BY_DEFAULT-}" ]]; then echo "$RES" | jq '.[0] | [ (.drvPath | test("'$NIX_STORE_DIR'.*\\.drv")), (.outputs.out | test("'$NIX_STORE_DIR'")) diff -Nru nix-2.8.0/tests/build-hook-ca-floating.nix nix-2.16.1+dfsg/tests/build-hook-ca-floating.nix --- nix-2.8.0/tests/build-hook-ca-floating.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/build-hook-ca-floating.nix 2023-06-06 10:55:23.000000000 +0000 @@ -1,53 +1,6 @@ { busybox }: -with import ./config.nix; - -let - - mkDerivation = args: - derivation ({ - inherit system; - builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - __contentAddressed = true; - } // removeAttrs args ["builder" "meta"]) - // { meta = args.meta or {}; }; - - input1 = mkDerivation { - shell = busybox; - name = "build-remote-input-1"; - buildCommand = "echo FOO > $out"; - requiredSystemFeatures = ["foo"]; - }; - - input2 = mkDerivation { - shell = busybox; - name = "build-remote-input-2"; - buildCommand = "echo BAR > $out"; - requiredSystemFeatures = ["bar"]; - }; - - input3 = mkDerivation { - shell = busybox; - name = "build-remote-input-3"; - buildCommand = '' - read x < ${input2} - echo $x BAZ > $out - ''; - requiredSystemFeatures = ["baz"]; - }; - -in - - mkDerivation { - shell = busybox; - name = "build-remote"; - buildCommand = - '' - read x < ${input1} - read y < ${input3} - echo "$x $y" > $out - ''; - } +import ./build-hook.nix { + inherit busybox; + contentAddressed = true; +} diff -Nru nix-2.8.0/tests/build-hook.nix nix-2.16.1+dfsg/tests/build-hook.nix --- nix-2.8.0/tests/build-hook.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/build-hook.nix 2023-06-06 10:55:23.000000000 +0000 @@ -1,15 +1,22 @@ -{ busybox }: +{ busybox, contentAddressed ? false }: with import ./config.nix; let + caArgs = if contentAddressed then { + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + __contentAddressed = true; + } else {}; + mkDerivation = args: derivation ({ inherit system; builder = busybox; args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; - } // removeAttrs args ["builder" "meta" "passthru"]) + } // removeAttrs args ["builder" "meta" "passthru"] + // caArgs) // { meta = args.meta or {}; passthru = args.passthru or {}; }; input1 = mkDerivation { diff -Nru nix-2.8.0/tests/build-remote-content-addressed-floating.sh nix-2.16.1+dfsg/tests/build-remote-content-addressed-floating.sh --- nix-2.8.0/tests/build-remote-content-addressed-floating.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/build-remote-content-addressed-floating.sh 2023-06-06 10:55:23.000000000 +0000 @@ -2,7 +2,7 @@ file=build-hook-ca-floating.nix -enableFeatures "ca-derivations ca-references" +enableFeatures "ca-derivations" CONTENT_ADDRESSED=true diff -Nru nix-2.8.0/tests/build-remote.sh nix-2.16.1+dfsg/tests/build-remote.sh --- nix-2.8.0/tests/build-remote.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/build-remote.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,5 +1,5 @@ -if ! canUseSandbox; then exit 99; fi -if ! [[ $busybox =~ busybox ]]; then exit 99; fi +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR unset NIX_STATE_DIR @@ -7,7 +7,7 @@ function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; } EXTRA_SYSTEM_FEATURES=() -if [[ -n "$CONTENT_ADDRESSED" ]]; then +if [[ -n "${CONTENT_ADDRESSED-}" ]]; then EXTRA_SYSTEM_FEATURES=("ca-derivations") fi @@ -34,36 +34,43 @@ grep 'FOO BAR BAZ' $TEST_ROOT/machine0/$outPath -set -o pipefail +testPrintOutPath=$(nix build -L -v -f $file --no-link --print-out-paths --max-jobs 0 \ + --arg busybox $busybox \ + --store $TEST_ROOT/machine0 \ + --builders "$(join_by '; ' "${builders[@]}")" +) + +[[ $testPrintOutPath =~ store.*build-remote ]] # Ensure that input1 was built on store1 due to the required feature. -nix path-info --store $TEST_ROOT/machine1 --all \ - | grep builder-build-remote-input-1.sh \ - | grep -v builder-build-remote-input-2.sh \ - | grep -v builder-build-remote-input-3.sh +output=$(nix path-info --store $TEST_ROOT/machine1 --all) +echo "$output" | grepQuiet builder-build-remote-input-1.sh +echo "$output" | grepQuietInverse builder-build-remote-input-2.sh +echo "$output" | grepQuietInverse builder-build-remote-input-3.sh +unset output # Ensure that input2 was built on store2 due to the required feature. -nix path-info --store $TEST_ROOT/machine2 --all \ - | grep -v builder-build-remote-input-1.sh \ - | grep builder-build-remote-input-2.sh \ - | grep -v builder-build-remote-input-3.sh +output=$(nix path-info --store $TEST_ROOT/machine2 --all) +echo "$output" | grepQuietInverse builder-build-remote-input-1.sh +echo "$output" | grepQuiet builder-build-remote-input-2.sh +echo "$output" | grepQuietInverse builder-build-remote-input-3.sh +unset output # Ensure that input3 was built on store3 due to the required feature. -nix path-info --store $TEST_ROOT/machine3 --all \ - | grep -v builder-build-remote-input-1.sh \ - | grep -v builder-build-remote-input-2.sh \ - | grep builder-build-remote-input-3.sh +output=$(nix path-info --store $TEST_ROOT/machine3 --all) +echo "$output" | grepQuietInverse builder-build-remote-input-1.sh +echo "$output" | grepQuietInverse builder-build-remote-input-2.sh +echo "$output" | grepQuiet builder-build-remote-input-3.sh +unset output -# Temporarily disabled because of https://github.com/NixOS/nix/issues/6209 -if [[ -z "$CONTENT_ADDRESSED" ]]; then - for i in input1 input3; do - nix log --store $TEST_ROOT/machine0 --file "$file" --arg busybox $busybox passthru."$i" | grep hi-$i - done -fi +for i in input1 input3; do +nix log --store $TEST_ROOT/machine0 --file "$file" --arg busybox $busybox passthru."$i" | grep hi-$i +done # Behavior of keep-failed out="$(nix-build 2>&1 failing.nix \ + --no-out-link \ --builders "$(join_by '; ' "${builders[@]}")" \ --keep-failed \ --store $TEST_ROOT/machine0 \ diff -Nru nix-2.8.0/tests/build-remote-trustless-after.sh nix-2.16.1+dfsg/tests/build-remote-trustless-after.sh --- nix-2.8.0/tests/build-remote-trustless-after.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/build-remote-trustless-after.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,2 @@ +outPath=$(readlink -f $TEST_ROOT/result) +grep 'FOO BAR BAZ' ${remoteDir}/${outPath} diff -Nru nix-2.8.0/tests/build-remote-trustless.sh nix-2.16.1+dfsg/tests/build-remote-trustless.sh --- nix-2.8.0/tests/build-remote-trustless.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/build-remote-trustless.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,14 @@ +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +remoteDir=$TEST_ROOT/remote + +# Note: ssh{-ng}://localhost bypasses ssh. See tests/build-remote.sh for +# more details. +nix-build $file -o $TEST_ROOT/result --max-jobs 0 \ + --arg busybox $busybox \ + --store $TEST_ROOT/local \ + --builders "$proto://localhost?remote-program=$prog&remote-store=${remoteDir}%3Fsystem-features=foo%20bar%20baz - - 1 1 foo,bar,baz" diff -Nru nix-2.8.0/tests/build-remote-trustless-should-fail-0.sh nix-2.16.1+dfsg/tests/build-remote-trustless-should-fail-0.sh --- nix-2.8.0/tests/build-remote-trustless-should-fail-0.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/build-remote-trustless-should-fail-0.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,29 @@ +source common.sh + +enableFeatures "daemon-trust-override" + +restartDaemon + +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +# We first build a dependency of the derivation we eventually want to +# build. +nix-build build-hook.nix -A passthru.input2 \ + -o "$TEST_ROOT/input2" \ + --arg busybox "$busybox" \ + --store "$TEST_ROOT/local" \ + --option system-features bar + +# Now when we go to build that downstream derivation, Nix will try to +# copy our already-build `input2` to the remote store. That store object +# is input-addressed, so this will fail. + +file=build-hook.nix +prog=$(readlink -e ./nix-daemon-untrusting.sh) +proto=ssh-ng + +expectStderr 1 source build-remote-trustless.sh \ + | grepQuiet "cannot add path '[^ ]*' because it lacks a signature by a trusted key" diff -Nru nix-2.8.0/tests/build-remote-trustless-should-pass-0.sh nix-2.16.1+dfsg/tests/build-remote-trustless-should-pass-0.sh --- nix-2.8.0/tests/build-remote-trustless-should-pass-0.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/build-remote-trustless-should-pass-0.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,9 @@ +source common.sh + +# Remote trusts us +file=build-hook.nix +prog=nix-store +proto=ssh + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff -Nru nix-2.8.0/tests/build-remote-trustless-should-pass-1.sh nix-2.16.1+dfsg/tests/build-remote-trustless-should-pass-1.sh --- nix-2.8.0/tests/build-remote-trustless-should-pass-1.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/build-remote-trustless-should-pass-1.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,9 @@ +source common.sh + +# Remote trusts us +file=build-hook.nix +prog=nix-daemon +proto=ssh-ng + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff -Nru nix-2.8.0/tests/build-remote-trustless-should-pass-2.sh nix-2.16.1+dfsg/tests/build-remote-trustless-should-pass-2.sh --- nix-2.8.0/tests/build-remote-trustless-should-pass-2.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/build-remote-trustless-should-pass-2.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,13 @@ +source common.sh + +enableFeatures "daemon-trust-override" + +restartDaemon + +# Remote doesn't trust us +file=build-hook.nix +prog=$(readlink -e ./nix-daemon-untrusting.sh) +proto=ssh-ng + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff -Nru nix-2.8.0/tests/build-remote-trustless-should-pass-3.sh nix-2.16.1+dfsg/tests/build-remote-trustless-should-pass-3.sh --- nix-2.8.0/tests/build-remote-trustless-should-pass-3.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/build-remote-trustless-should-pass-3.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,14 @@ +source common.sh + +enableFeatures "daemon-trust-override" + +restartDaemon + +# Remote doesn't trusts us, but this is fine because we are only +# building (fixed) CA derivations. +file=build-hook-ca-fixed.nix +prog=$(readlink -e ./nix-daemon-untrusting.sh) +proto=ssh-ng + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff -Nru nix-2.8.0/tests/build.sh nix-2.16.1+dfsg/tests/build.sh --- nix-2.8.0/tests/build.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/build.sh 2023-06-06 10:55:23.000000000 +0000 @@ -2,30 +2,130 @@ clearStore -# Make sure that 'nix build' only returns the outputs we asked for. -nix build -f multiple-outputs.nix --json a --no-link | jq --exit-status ' +# Make sure that 'nix build' returns all outputs by default. +nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status ' (.[0] | (.drvPath | match(".*multiple-outputs-a.drv")) and - (.outputs | keys | length == 1) and - (.outputs.first | match(".*multiple-outputs-a-first"))) + (.outputs | + (keys | length == 2) and + (.first | match(".*multiple-outputs-a-first")) and + (.second | match(".*multiple-outputs-a-second")))) + and (.[1] | + (.drvPath | match(".*multiple-outputs-b.drv")) and + (.outputs | + (keys | length == 1) and + (.out | match(".*multiple-outputs-b")))) ' -nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status ' +# Test output selection using the '^' syntax. +nix build -f multiple-outputs.nix --json a^first --no-link | jq --exit-status ' (.[0] | (.drvPath | match(".*multiple-outputs-a.drv")) and - (.outputs | keys | length == 2) and - (.outputs.first | match(".*multiple-outputs-a-first")) and - (.outputs.second | match(".*multiple-outputs-a-second"))) - and (.[1] | - (.drvPath | match(".*multiple-outputs-b.drv")) and - (.outputs | keys | length == 1) and - (.outputs.out | match(".*multiple-outputs-b"))) + (.outputs | keys == ["first"])) +' + +nix build -f multiple-outputs.nix --json a^second,first --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | keys == ["first", "second"])) +' + +nix build -f multiple-outputs.nix --json 'a^*' --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | keys == ["first", "second"])) ' -testNormalization () { - clearStore - outPath=$(nix-build ./simple.nix --no-out-link) - test "$(stat -c %Y $outPath)" -eq 1 -} +# Test that 'outputsToInstall' is respected by default. +nix build -f multiple-outputs.nix --json e --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a", "b"])) +' + +# But not when it's overriden. +nix build -f multiple-outputs.nix --json e^a_a --no-link +nix build -f multiple-outputs.nix --json e^a_a --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a"])) +' + +nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a", "b", "c"])) +' + +# test buidling from non-drv attr path + +nix build -f multiple-outputs.nix --json 'e.a_a.outPath' --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a"])) +' -testNormalization +# Illegal type of string context +expectStderr 1 nix build -f multiple-outputs.nix 'e.a_a.drvPath' \ + | grepQuiet "has a context which refers to a complete source and binary closure." + +# No string context +expectStderr 1 nix build --expr '""' --no-link \ + | grepQuiet "has 0 entries in its context. It should only have exactly one entry" + +# Too much string context +expectStderr 1 nix build --impure --expr 'with (import ./multiple-outputs.nix).e.a_a; "${drvPath}${outPath}"' --no-link \ + | grepQuiet "has 2 entries in its context. It should only have exactly one entry" + +nix build --impure --json --expr 'builtins.unsafeDiscardOutputDependency (import ./multiple-outputs.nix).e.a_a.drvPath' --no-link | jq --exit-status ' + (.[0] | .path | match(".*multiple-outputs-e.drv")) +' + +# Test building from raw store path to drv not expression. + +drv=$(nix eval -f multiple-outputs.nix --raw a.drvPath) +if nix build "$drv^not-an-output" --no-link --json; then + fail "'not-an-output' should fail to build" +fi + +if nix build "$drv^" --no-link --json; then + fail "'empty outputs list' should fail to build" +fi + +if nix build "$drv^*nope" --no-link --json; then + fail "'* must be entire string' should fail to build" +fi + +nix build "$drv^first" --no-link --json | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | + (keys | length == 1) and + (.first | match(".*multiple-outputs-a-first")) and + (has("second") | not))) +' + +nix build "$drv^first,second" --no-link --json | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | + (keys | length == 2) and + (.first | match(".*multiple-outputs-a-first")) and + (.second | match(".*multiple-outputs-a-second")))) +' + +nix build "$drv^*" --no-link --json | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | + (keys | length == 2) and + (.first | match(".*multiple-outputs-a-first")) and + (.second | match(".*multiple-outputs-a-second")))) +' + +# Make sure that `--impure` works (regression test for https://github.com/NixOS/nix/issues/6488) +nix build --impure -f multiple-outputs.nix --json e --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a", "b"])) +' diff -Nru nix-2.8.0/tests/ca/build.sh nix-2.16.1+dfsg/tests/ca/build.sh --- nix-2.8.0/tests/ca/build.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/ca/build.sh 2023-06-06 10:55:23.000000000 +0000 @@ -2,14 +2,14 @@ source common.sh -drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 1) -nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1 +drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1) +nix derivation show "$drv" --arg seed 1 buildAttr () { local derivationPath=$1 local seedValue=$2 shift; shift - local args=("--experimental-features" "ca-derivations" "./content-addressed.nix" "-A" "$derivationPath" --arg seed "$seedValue" "--no-out-link") + local args=("./content-addressed.nix" "-A" "$derivationPath" --arg seed "$seedValue" "--no-out-link") args+=("$@") nix-build "${args[@]}" } @@ -19,7 +19,7 @@ local outPath=$(buildAttr dependentNonCA 1) nix copy --to file://$cacheDir $outPath clearStore - buildAttr dependentNonCA 1 --option substituters file://$cacheDir --no-require-sigs |& (! grep "building dependent-non-ca") + buildAttr dependentNonCA 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse "building dependent-non-ca" } testDeterministicCA () { @@ -46,17 +46,17 @@ } testGC () { - nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5 - nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true + nix-instantiate ./content-addressed.nix -A rootCA --arg seed 5 + nix-collect-garbage --option keep-derivations true clearStore buildAttr rootCA 1 --out-link $TEST_ROOT/rootCA - nix-collect-garbage --experimental-features ca-derivations + nix-collect-garbage buildAttr rootCA 1 -j0 } testNixCommand () { clearStore - nix build --experimental-features 'nix-command ca-derivations' --file ./content-addressed.nix --no-link + nix build --file ./content-addressed.nix --no-link } # Regression test for https://github.com/NixOS/nix/issues/4775 diff -Nru nix-2.8.0/tests/ca/common.sh nix-2.16.1+dfsg/tests/ca/common.sh --- nix-2.8.0/tests/ca/common.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/ca/common.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,5 +1,5 @@ source ../common.sh -enableFeatures "ca-derivations ca-references" +enableFeatures "ca-derivations" restartDaemon diff -Nru nix-2.8.0/tests/ca/content-addressed.nix nix-2.16.1+dfsg/tests/ca/content-addressed.nix --- nix-2.8.0/tests/ca/content-addressed.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/ca/content-addressed.nix 2023-06-06 10:55:23.000000000 +0000 @@ -23,7 +23,7 @@ }; rootCA = mkCADerivation { name = "rootCA"; - outputs = [ "out" "dev" "foo"]; + outputs = [ "out" "dev" "foo" ]; buildCommand = '' echo "building a CA derivation" echo "The seed is ${toString seed}" @@ -75,7 +75,7 @@ buildCommand = '' mkdir -p $out/bin echo ${rootCA} # Just to make it depend on it - echo "" > $out/bin/${name} + echo "#! ${shell}" > $out/bin/${name} chmod +x $out/bin/${name} ''; }; diff -Nru nix-2.8.0/tests/ca/derivation-json.sh nix-2.16.1+dfsg/tests/ca/derivation-json.sh --- nix-2.8.0/tests/ca/derivation-json.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/ca/derivation-json.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,29 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +drvPath=$(nix-instantiate ../simple.nix) + +nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json + +drvPath2=$(nix derivation add < $TEST_HOME/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Content-addressed derivations can be renamed. +jq '.name = "foo"' < $TEST_HOME/simple.json > $TEST_HOME/foo.json +drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json) +# With --dry-run nothing is actually written +[[ ! -e "$drvPath3" ]] + +# But the JSON is rejected without the experimental feature +expectStderr 1 nix derivation add < $TEST_HOME/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled" + +# Without --dry-run it is actually written +drvPath4=$(nix derivation add < $TEST_HOME/foo.json) +[[ "$drvPath4" = "$drvPath3" ]] +[[ -e "$drvPath3" ]] + +# The modified derivation read back as JSON matches +nix derivation show $drvPath3 | jq .[] > $TEST_HOME/foo-read.json +diff $TEST_HOME/foo.json $TEST_HOME/foo-read.json diff -Nru nix-2.8.0/tests/ca/new-build-cmd.sh nix-2.16.1+dfsg/tests/ca/new-build-cmd.sh --- nix-2.8.0/tests/ca/new-build-cmd.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/ca/new-build-cmd.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,5 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 +cd .. +source ./build.sh diff -Nru nix-2.8.0/tests/ca/recursive.sh nix-2.16.1+dfsg/tests/ca/recursive.sh --- nix-2.8.0/tests/ca/recursive.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/ca/recursive.sh 2023-06-06 10:55:23.000000000 +0000 @@ -7,5 +7,3 @@ export NIX_TESTS_CA_BY_DEFAULT=1 cd .. source ./recursive.sh - - diff -Nru nix-2.8.0/tests/ca/selfref-gc.sh nix-2.16.1+dfsg/tests/ca/selfref-gc.sh --- nix-2.8.0/tests/ca/selfref-gc.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/ca/selfref-gc.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source common.sh + +requireDaemonNewerThan "2.4pre20210626" + +enableFeatures "ca-derivations nix-command flakes" + +export NIX_TESTS_CA_BY_DEFAULT=1 +cd .. +source ./selfref-gc.sh diff -Nru nix-2.8.0/tests/ca/substitute.sh nix-2.16.1+dfsg/tests/ca/substitute.sh --- nix-2.8.0/tests/ca/substitute.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/ca/substitute.sh 2023-06-06 10:55:23.000000000 +0000 @@ -25,7 +25,14 @@ # Check that the thing we’ve just substituted has its realisation stored nix realisation info --file ./content-addressed.nix transitivelyDependentCA # Check that its dependencies have it too -nix realisation info --file ./content-addressed.nix dependentCA rootCA +nix realisation info --file ./content-addressed.nix dependentCA +# nix realisation info --file ./content-addressed.nix rootCA --outputs out + +if isDaemonNewer "2.13"; then + pushToStore="../push-to-store.sh" +else + pushToStore="../push-to-store-old.sh" +fi # Same thing, but # 1. With non-ca derivations @@ -36,7 +43,7 @@ # # Regression test for #4725 clearStore -nix build --file ../simple.nix -L --no-link --post-build-hook ../push-to-store.sh +nix build --file ../simple.nix -L --no-link --post-build-hook "$pushToStore" clearStore rm -r "$REMOTE_STORE_DIR/realisations" nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 @@ -51,7 +58,7 @@ fi # Test the local realisation disk cache -buildDrvs --post-build-hook ../push-to-store.sh +buildDrvs --post-build-hook "$pushToStore" clearStore # Add the realisations of rootCA to the cachecache clearCacheCache diff -Nru nix-2.8.0/tests/ca/why-depends.sh nix-2.16.1+dfsg/tests/ca/why-depends.sh --- nix-2.8.0/tests/ca/why-depends.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/ca/why-depends.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,5 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +cd .. && source why-depends.sh diff -Nru nix-2.8.0/tests/ca-shell.nix nix-2.16.1+dfsg/tests/ca-shell.nix --- nix-2.8.0/tests/ca-shell.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/ca-shell.nix 2023-06-06 10:55:23.000000000 +0000 @@ -1 +1 @@ -{ ... }@args: import ./shell.nix (args // { contentAddressed = true; }) +{ inNixShell ? false, ... }@args: import ./shell.nix (args // { contentAddressed = true; }) diff -Nru nix-2.8.0/tests/check.nix nix-2.16.1+dfsg/tests/check.nix --- nix-2.8.0/tests/check.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/check.nix 2023-06-06 10:55:23.000000000 +0000 @@ -44,7 +44,7 @@ }; hashmismatch = import { - url = "file://" + builtins.getEnv "TMPDIR" + "/dummy"; + url = "file://" + builtins.getEnv "TEST_ROOT" + "/dummy"; sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"; }; diff -Nru nix-2.8.0/tests/check-refs.nix nix-2.16.1+dfsg/tests/check-refs.nix --- nix-2.8.0/tests/check-refs.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/check-refs.nix 2023-06-06 10:55:23.000000000 +0000 @@ -67,4 +67,11 @@ disallowedReferences = [test5]; }; + test11 = makeTest 11 { + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + outputChecks.out.allowedReferences = []; + buildCommand = ''echo ${dep} > "''${outputs[out]}"''; + }; + } diff -Nru nix-2.8.0/tests/check-refs.sh nix-2.16.1+dfsg/tests/check-refs.sh --- nix-2.8.0/tests/check-refs.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/check-refs.sh 2023-06-06 10:55:23.000000000 +0000 @@ -8,14 +8,14 @@ # test1 references dep, not itself. test1=$(nix-build -o $RESULT check-refs.nix -A test1) -(! nix-store -q --references $test1 | grep -q $test1) -nix-store -q --references $test1 | grep -q $dep +nix-store -q --references $test1 | grepQuietInverse $test1 +nix-store -q --references $test1 | grepQuiet $dep # test2 references src, not itself nor dep. test2=$(nix-build -o $RESULT check-refs.nix -A test2) -(! nix-store -q --references $test2 | grep -q $test2) -(! nix-store -q --references $test2 | grep -q $dep) -nix-store -q --references $test2 | grep -q aux-ref +nix-store -q --references $test2 | grepQuietInverse $test2 +nix-store -q --references $test2 | grepQuietInverse $dep +nix-store -q --references $test2 | grepQuiet aux-ref # test3 should fail (unallowed ref). (! nix-build -o $RESULT check-refs.nix -A test3) @@ -40,3 +40,12 @@ # test10 should succeed (no disallowed references). nix-build -o $RESULT check-refs.nix -A test10 + +if isDaemonNewer 2.12pre20230103; then + enableFeatures discard-references + restartDaemon + + # test11 should succeed. + test11=$(nix-build -o $RESULT check-refs.nix -A test11) + [[ -z $(nix-store -q --references "$test11") ]] +fi diff -Nru nix-2.8.0/tests/check-reqs.sh nix-2.16.1+dfsg/tests/check-reqs.sh --- nix-2.8.0/tests/check-reqs.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/check-reqs.sh 2023-06-06 10:55:23.000000000 +0000 @@ -8,8 +8,8 @@ (! nix-build -o $RESULT check-reqs.nix -A test2) (! nix-build -o $RESULT check-reqs.nix -A test3) -(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grep -q 'check-reqs-dep1' -(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grep -q 'check-reqs-dep2' +(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep1' +(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep2' (! nix-build -o $RESULT check-reqs.nix -A test5) (! nix-build -o $RESULT check-reqs.nix -A test6) diff -Nru nix-2.8.0/tests/check.sh nix-2.16.1+dfsg/tests/check.sh --- nix-2.8.0/tests/check.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/check.sh 2023-06-06 10:55:23.000000000 +0000 @@ -37,7 +37,7 @@ nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ --no-out-link --check --keep-failed 2> $TEST_ROOT/log -if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi +if grepQuiet 'may not be deterministic' $TEST_ROOT/log; then false; fi checkBuildTempDirRemoved $TEST_ROOT/log nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ @@ -58,12 +58,6 @@ clearStore -nix-build dependencies.nix --no-out-link --repeat 3 - -nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log || status=$? -[ "$status" = "1" ] -grep 'differs from previous round' $TEST_ROOT/log - path=$(nix-build check.nix -A fetchurl --no-out-link) chmod +w $path @@ -77,13 +71,13 @@ nix-build check.nix -A fetchurl --no-out-link --repair [[ $(cat $path) != foo ]] -echo 'Hello World' > $TMPDIR/dummy +echo 'Hello World' > $TEST_ROOT/dummy nix-build check.nix -A hashmismatch --no-out-link || status=$? [ "$status" = "102" ] -echo -n > $TMPDIR/dummy +echo -n > $TEST_ROOT/dummy nix-build check.nix -A hashmismatch --no-out-link -echo 'Hello World' > $TMPDIR/dummy +echo 'Hello World' > $TEST_ROOT/dummy nix-build check.nix -A hashmismatch --no-out-link --check || status=$? [ "$status" = "102" ] diff -Nru nix-2.8.0/tests/common/vars-and-functions.sh.in nix-2.16.1+dfsg/tests/common/vars-and-functions.sh.in --- nix-2.8.0/tests/common/vars-and-functions.sh.in 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/common/vars-and-functions.sh.in 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,276 @@ +set -eu -o pipefail + +if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then + +COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 + +export PS4='+(${BASH_SOURCE[0]}:$LINENO) ' + +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} +export NIX_STORE_DIR +if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then + # Maybe the build directory is symlinked. + export NIX_IGNORE_SYMLINK_STORE=1 + NIX_STORE_DIR=$TEST_ROOT/store +fi +export NIX_LOCALSTATE_DIR=$TEST_ROOT/var +export NIX_LOG_DIR=$TEST_ROOT/var/log/nix +export NIX_STATE_DIR=$TEST_ROOT/var/nix +export NIX_CONF_DIR=$TEST_ROOT/etc +export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket +unset NIX_USER_CONF_FILES +export _NIX_TEST_SHARED=$TEST_ROOT/shared +if [[ -n $NIX_STORE ]]; then + export _NIX_TEST_NO_SANDBOX=1 +fi +export _NIX_IN_TEST=$TEST_ROOT/shared +export _NIX_TEST_NO_LSOF=1 +export NIX_REMOTE=${NIX_REMOTE_-} +unset NIX_PATH +export TEST_HOME=$TEST_ROOT/test-home +export HOME=$TEST_HOME +unset XDG_STATE_HOME +unset XDG_DATA_HOME +unset XDG_CONFIG_HOME +unset XDG_CONFIG_DIRS +unset XDG_CACHE_HOME +mkdir -p $TEST_HOME + +export PATH=@bindir@:$PATH +if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then + export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH +fi +DAEMON_PATH="$PATH" +if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then + DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH" +fi +coreutils=@coreutils@ + +export dot=@dot@ +export SHELL="@bash@" +export PAGER=cat +export busybox="@sandbox_shell@" + +export version=@PACKAGE_VERSION@ +export system=@system@ + +export BUILD_SHARED_LIBS=@BUILD_SHARED_LIBS@ + +export IMPURE_VAR1=foo +export IMPURE_VAR2=bar + +cacheDir=$TEST_ROOT/binary-cache + +readLink() { + ls -l "$1" | sed 's/.*->\ //' +} + +clearProfiles() { + profiles="$HOME"/.local/state/nix/profiles + rm -rf "$profiles" +} + +clearStore() { + echo "clearing store..." + chmod -R +w "$NIX_STORE_DIR" + rm -rf "$NIX_STORE_DIR" + mkdir "$NIX_STORE_DIR" + rm -rf "$NIX_STATE_DIR" + mkdir "$NIX_STATE_DIR" + clearProfiles +} + +clearCache() { + rm -rf "$cacheDir" +} + +clearCacheCache() { + rm -f $TEST_HOME/.cache/nix/binary-cache* +} + +startDaemon() { + # Don’t start the daemon twice, as this would just make it loop indefinitely + if [[ "${_NIX_TEST_DAEMON_PID-}" != '' ]]; then + return + fi + # Start the daemon, wait for the socket to appear. + rm -f $NIX_DAEMON_SOCKET_PATH + PATH=$DAEMON_PATH nix-daemon & + _NIX_TEST_DAEMON_PID=$! + export _NIX_TEST_DAEMON_PID + for ((i = 0; i < 300; i++)); do + if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then + DAEMON_STARTED=1 + break; + fi + sleep 0.1 + done + if [[ -z ${DAEMON_STARTED+x} ]]; then + fail "Didn’t manage to start the daemon" + fi + trap "killDaemon" EXIT + # Save for if daemon is killed + NIX_REMOTE_OLD=$NIX_REMOTE + export NIX_REMOTE=daemon +} + +killDaemon() { + # Don’t fail trying to stop a non-existant daemon twice + if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then + return + fi + kill $_NIX_TEST_DAEMON_PID + for i in {0..100}; do + kill -0 $_NIX_TEST_DAEMON_PID 2> /dev/null || break + sleep 0.1 + done + kill -9 $_NIX_TEST_DAEMON_PID 2> /dev/null || true + wait $_NIX_TEST_DAEMON_PID || true + rm -f $NIX_DAEMON_SOCKET_PATH + # Indicate daemon is stopped + unset _NIX_TEST_DAEMON_PID + # Restore old nix remote + NIX_REMOTE=$NIX_REMOTE_OLD + trap "" EXIT +} + +restartDaemon() { + [[ -z "${_NIX_TEST_DAEMON_PID:-}" ]] && return 0 + + killDaemon + startDaemon +} + +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then + _canUseSandbox=1 +fi + +isDaemonNewer () { + [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 + local requiredVersion="$1" + local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3) + [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] +} + +skipTest () { + echo "$1, skipping this test..." >&2 + exit 99 +} + +requireDaemonNewerThan () { + isDaemonNewer "$1" || skipTest "Daemon is too old" +} + +canUseSandbox() { + [[ ${_canUseSandbox-} ]] +} + +requireSandboxSupport () { + canUseSandbox || skipTest "Sandboxing not supported" +} + +requireGit() { + [[ $(type -p git) ]] || skipTest "Git not installed" +} + +fail() { + echo "$1" >&2 + exit 1 +} + +# Run a command failing if it didn't exit with the expected exit code. +# +# Has two advantages over the built-in `!`: +# +# 1. `!` conflates all non-0 codes. `expect` allows testing for an exact +# code. +# +# 2. `!` unexpectedly negates `set -e`, and cannot be used on individual +# pipeline stages with `set -o pipefail`. It only works on the entire +# pipeline, which is useless if we want, say, `nix ...` invocation to +# *fail*, but a grep on the error message it outputs to *succeed*. +expect() { + local expected res + expected="$1" + shift + "$@" && res=0 || res="$?" + if [[ $res -ne $expected ]]; then + echo "Expected '$expected' but got '$res' while running '${*@Q}'" >&2 + return 1 + fi + return 0 +} + +# Better than just doing `expect ... >&2` because the "Expected..." +# message below will *not* be redirected. +expectStderr() { + local expected res + expected="$1" + shift + "$@" 2>&1 && res=0 || res="$?" + if [[ $res -ne $expected ]]; then + echo "Expected '$expected' but got '$res' while running '${*@Q}'" >&2 + return 1 + fi + return 0 +} + +needLocalStore() { + if [[ "$NIX_REMOTE" == "daemon" ]]; then + skipTest "Can’t run through the daemon ($1)" + fi +} + +# Just to make it easy to find which tests should be fixed +buggyNeedLocalStore() { + needLocalStore "$1" +} + +enableFeatures() { + local features="$1" + sed -i 's/experimental-features .*/& '"$features"'/' "$NIX_CONF_DIR"/nix.conf +} + +set -x + +onError() { + set +x + echo "$0: test failed at:" >&2 + for ((i = 1; i < ${#BASH_SOURCE[@]}; i++)); do + if [[ -z ${BASH_SOURCE[i]} ]]; then break; fi + echo " ${FUNCNAME[i]} in ${BASH_SOURCE[i]}:${BASH_LINENO[i-1]}" >&2 + done +} + +# `grep -v` doesn't work well for exit codes. We want `!(exist line l. l +# matches)`. It gives us `exist line l. !(l matches)`. +# +# `!` normally doesn't work well with `set -e`, but when we wrap in a +# function it *does*. +grepInverse() { + ! grep "$@" +} + +# A shorthand, `> /dev/null` is a bit noisy. +# +# `grep -q` would seem to do this, no function necessary, but it is a +# bad fit with pipes and `set -o pipefail`: `-q` will exit after the +# first match, and then subsequent writes will result in broken pipes. +# +# Note that reproducing the above is a bit tricky as it depends on +# non-deterministic properties such as the timing between the match and +# the closing of the pipe, the buffering of the pipe, and the speed of +# the producer into the pipe. But rest assured we've seen it happen in +# CI reliably. +grepQuiet() { + grep "$@" > /dev/null +} + +# The previous two, combined +grepQuietInverse() { + ! grep "$@" > /dev/null +} + +trap onError ERR + +fi # COMMON_VARS_AND_FUNCTIONS_SH_SOURCED diff -Nru nix-2.8.0/tests/common.sh nix-2.16.1+dfsg/tests/common.sh --- nix-2.8.0/tests/common.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/common.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,12 @@ +set -eu -o pipefail + +if [[ -z "${COMMON_SH_SOURCED-}" ]]; then + +COMMON_SH_SOURCED=1 + +source "$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")/common/vars-and-functions.sh" +if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then + startDaemon +fi + +fi # COMMON_SH_SOURCED diff -Nru nix-2.8.0/tests/common.sh.in nix-2.16.1+dfsg/tests/common.sh.in --- nix-2.8.0/tests/common.sh.in 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/common.sh.in 1970-01-01 00:00:00.000000000 +0000 @@ -1,190 +0,0 @@ -set -e - -if [[ -z "$COMMON_SH_SOURCED" ]]; then - -COMMON_SH_SOURCED=1 - -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} -export NIX_STORE_DIR -if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then - # Maybe the build directory is symlinked. - export NIX_IGNORE_SYMLINK_STORE=1 - NIX_STORE_DIR=$TEST_ROOT/store -fi -export NIX_LOCALSTATE_DIR=$TEST_ROOT/var -export NIX_LOG_DIR=$TEST_ROOT/var/log/nix -export NIX_STATE_DIR=$TEST_ROOT/var/nix -export NIX_CONF_DIR=$TEST_ROOT/etc -export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket -unset NIX_USER_CONF_FILES -export _NIX_TEST_SHARED=$TEST_ROOT/shared -if [[ -n $NIX_STORE ]]; then - export _NIX_TEST_NO_SANDBOX=1 -fi -export _NIX_IN_TEST=$TEST_ROOT/shared -export _NIX_TEST_NO_LSOF=1 -export NIX_REMOTE=$NIX_REMOTE_ -unset NIX_PATH -export TEST_HOME=$TEST_ROOT/test-home -export HOME=$TEST_HOME -unset XDG_CONFIG_HOME -unset XDG_CONFIG_DIRS -unset XDG_CACHE_HOME -mkdir -p $TEST_HOME - -export PATH=@bindir@:$PATH -if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then - export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH -fi -DAEMON_PATH="$PATH" -if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then - DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH" -fi -coreutils=@coreutils@ - -export dot=@dot@ -export SHELL="@bash@" -export PAGER=cat -export busybox="@sandbox_shell@" - -export version=@PACKAGE_VERSION@ -export system=@system@ - -export IMPURE_VAR1=foo -export IMPURE_VAR2=bar - -cacheDir=$TEST_ROOT/binary-cache - -readLink() { - ls -l "$1" | sed 's/.*->\ //' -} - -clearProfiles() { - profiles="$NIX_STATE_DIR"/profiles - rm -rf $profiles -} - -clearStore() { - echo "clearing store..." - chmod -R +w "$NIX_STORE_DIR" - rm -rf "$NIX_STORE_DIR" - mkdir "$NIX_STORE_DIR" - rm -rf "$NIX_STATE_DIR" - mkdir "$NIX_STATE_DIR" - clearProfiles -} - -clearCache() { - rm -rf "$cacheDir" -} - -clearCacheCache() { - rm -f $TEST_HOME/.cache/nix/binary-cache* -} - -startDaemon() { - # Don’t start the daemon twice, as this would just make it loop indefinitely - if [[ "$NIX_REMOTE" == daemon ]]; then - return - fi - # Start the daemon, wait for the socket to appear. - rm -f $NIX_DAEMON_SOCKET_PATH - PATH=$DAEMON_PATH nix-daemon& - pidDaemon=$! - for ((i = 0; i < 300; i++)); do - if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then - DAEMON_STARTED=1 - break; - fi - sleep 0.1 - done - if [[ -z ${DAEMON_STARTED+x} ]]; then - fail "Didn’t manage to start the daemon" - fi - trap "killDaemon" EXIT - export NIX_REMOTE=daemon -} - -killDaemon() { - kill $pidDaemon - for i in {0..100}; do - kill -0 $pidDaemon 2> /dev/null || break - sleep 0.1 - done - kill -9 $pidDaemon 2> /dev/null || true - wait $pidDaemon || true - trap "" EXIT -} - -restartDaemon() { - [[ -z "${pidDaemon:-}" ]] && return 0 - - killDaemon - unset NIX_REMOTE - startDaemon -} - -if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then - _canUseSandbox=1 -fi - -isDaemonNewer () { - [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 - local requiredVersion="$1" - local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3) - [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] -} - -requireDaemonNewerThan () { - isDaemonNewer "$1" || exit 99 -} - -canUseSandbox() { - if [[ ! $_canUseSandbox ]]; then - echo "Sandboxing not supported, skipping this test..." - return 1 - fi - - return 0 -} - -fail() { - echo "$1" - exit 1 -} - -expect() { - local expected res - expected="$1" - shift - set +e - "$@" - res="$?" - set -e - [[ $res -eq $expected ]] -} - -needLocalStore() { - if [[ "$NIX_REMOTE" == "daemon" ]]; then - echo "Can’t run through the daemon ($1), skipping this test..." - return 99 - fi -} - -# Just to make it easy to find which tests should be fixed -buggyNeedLocalStore() { - needLocalStore -} - -enableFeatures() { - local features="$1" - sed -i 's/experimental-features .*/& '"$features"'/' "$NIX_CONF_DIR"/nix.conf -} - -set -x - -if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then - startDaemon -fi - -fi # COMMON_SH_SOURCED diff -Nru nix-2.8.0/tests/completions.sh nix-2.16.1+dfsg/tests/completions.sh --- nix-2.8.0/tests/completions.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/completions.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,68 @@ +source common.sh + +cd "$TEST_ROOT" + +mkdir -p dep +cat < dep/flake.nix +{ + outputs = i: { }; +} +EOF +mkdir -p foo +cat < foo/flake.nix +{ + inputs.a.url = "path:$(realpath dep)"; + + outputs = i: { + sampleOutput = 1; + }; +} +EOF +mkdir -p bar +cat < bar/flake.nix +{ + inputs.b.url = "path:$(realpath dep)"; + + outputs = i: { + sampleOutput = 1; + }; +} +EOF +mkdir -p err +cat < err/flake.nix +throw "error" +EOF + +# Test the completion of a subcommand +[[ "$(NIX_GET_COMPLETIONS=1 nix buil)" == $'normal\nbuild\t' ]] +[[ "$(NIX_GET_COMPLETIONS=2 nix flake metad)" == $'normal\nmetadata\t' ]] + +# Filename completion +[[ "$(NIX_GET_COMPLETIONS=2 nix build ./f)" == $'filenames\n./foo\t' ]] +[[ "$(NIX_GET_COMPLETIONS=2 nix build ./nonexistent)" == $'filenames' ]] + +# Input override completion +[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] +[[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]] +## With multiple input flakes +[[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] +## With tilde expansion +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] +## Out of order +[[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] + +# Cli flag completion +NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" + +# Config option completion +## With `--option` +NIX_GET_COMPLETIONS=3 nix build --option allow-import-from | grep -- "allow-import-from-derivation" +## As a cli flag – not working atm +# NIX_GET_COMPLETIONS=2 nix build --allow-import-from | grep -- "allow-import-from-derivation" + +# Attr path completions +[[ "$(NIX_GET_COMPLETIONS=2 nix eval ./foo\#sam)" == $'attrs\n./foo#sampleOutput\t' ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix eval --file ./foo/flake.nix outp)" == $'attrs\noutputs\t' ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix eval --file ./err/flake.nix outp 2>&1)" == $'attrs' ]] +[[ "$(NIX_GET_COMPLETIONS=2 nix eval ./err\# 2>&1)" == $'attrs' ]] diff -Nru nix-2.8.0/tests/compute-levels.sh nix-2.16.1+dfsg/tests/compute-levels.sh --- nix-2.8.0/tests/compute-levels.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/compute-levels.sh 2023-06-06 10:55:23.000000000 +0000 @@ -3,5 +3,5 @@ if [[ $(uname -ms) = "Linux x86_64" ]]; then # x86_64 CPUs must always support the baseline # microarchitecture level. - nix -vv --version | grep -q "x86_64-v1-linux" + nix -vv --version | grepQuiet "x86_64-v1-linux" fi diff -Nru nix-2.8.0/tests/config.sh nix-2.16.1+dfsg/tests/config.sh --- nix-2.8.0/tests/config.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/config.sh 2023-06-06 10:55:23.000000000 +0000 @@ -51,3 +51,8 @@ [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] [[ $exp_features == "flakes nix-command" ]] + +# Test that it's possible to retrieve a single setting's value +val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) +val2=$(nix show-config warn-dirty) +[[ $val == $val2 ]] diff -Nru nix-2.8.0/tests/db-migration.sh nix-2.16.1+dfsg/tests/db-migration.sh --- nix-2.8.0/tests/db-migration.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/db-migration.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,19 +1,18 @@ # Test that we can successfully migrate from an older db schema +source common.sh + # Only run this if we have an older Nix available # XXX: This assumes that the `daemon` package is older than the `client` one -if [[ -z "$NIX_DAEMON_PACKAGE" ]]; then - exit 99 +if [[ -z "${NIX_DAEMON_PACKAGE-}" ]]; then + skipTest "not using the Nix daemon" fi -source common.sh - killDaemon -unset NIX_REMOTE # Fill the db using the older Nix PATH_WITH_NEW_NIX="$PATH" -export PATH="$NIX_DAEMON_PACKAGE/bin:$PATH" +export PATH="${NIX_DAEMON_PACKAGE}/bin:$PATH" clearStore nix-build simple.nix --no-out-link nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 diff -Nru nix-2.8.0/tests/dependencies.sh nix-2.16.1+dfsg/tests/dependencies.sh --- nix-2.8.0/tests/dependencies.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/dependencies.sh 2023-06-06 10:55:23.000000000 +0000 @@ -36,10 +36,10 @@ echo "output closure contains $deps" # The output path should be in the closure. -echo "$deps" | grep -q "$outPath" +echo "$deps" | grepQuiet "$outPath" # Input-1 is not retained. -if echo "$deps" | grep -q "dependencies-input-1"; then exit 1; fi +if echo "$deps" | grepQuiet "dependencies-input-1"; then exit 1; fi # Input-2 is retained. input2OutPath=$(echo "$deps" | grep "dependencies-input-2") @@ -49,4 +49,4 @@ # Check that the derivers are set properly. test $(nix-store -q --deriver "$outPath") = "$drvPath" -nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" +nix-store -q --deriver "$input2OutPath" | grepQuiet -- "-input-2.drv" diff -Nru nix-2.8.0/tests/derivation-json.sh nix-2.16.1+dfsg/tests/derivation-json.sh --- nix-2.8.0/tests/derivation-json.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/derivation-json.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,12 @@ +source common.sh + +drvPath=$(nix-instantiate simple.nix) + +nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json + +drvPath2=$(nix derivation add < $TEST_HOME/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Input addressed derivations cannot be renamed. +jq '.name = "foo"' < $TEST_HOME/simple.json | expectStderr 1 nix derivation add | grepQuiet "has incorrect output" diff -Nru nix-2.8.0/tests/describe-stores.sh nix-2.16.1+dfsg/tests/describe-stores.sh --- nix-2.8.0/tests/describe-stores.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/describe-stores.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,8 +0,0 @@ -source common.sh - -# Query an arbitrary value in `nix describe-stores --json`'s output just to -# check that it has the right structure -[[ $(nix --experimental-features 'nix-command flakes' describe-stores --json | jq '.["SSH Store"]["compress"]["defaultValue"]') == false ]] - -# Ensure that the output of `nix describe-stores` isn't empty -[[ -n $(nix --experimental-features 'nix-command flakes' describe-stores) ]] diff -Nru nix-2.8.0/tests/dyn-drv/common.sh nix-2.16.1+dfsg/tests/dyn-drv/common.sh --- nix-2.8.0/tests/dyn-drv/common.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/dyn-drv/common.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,8 @@ +source ../common.sh + +# Need backend to support text-hashing too +requireDaemonNewerThan "2.16.0pre20230419" + +enableFeatures "ca-derivations dynamic-derivations" + +restartDaemon diff -Nru nix-2.8.0/tests/dyn-drv/config.nix.in nix-2.16.1+dfsg/tests/dyn-drv/config.nix.in --- nix-2.8.0/tests/dyn-drv/config.nix.in 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/dyn-drv/config.nix.in 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,27 @@ +let + contentAddressedByDefault = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT" == "1"; + caArgs = if contentAddressedByDefault then { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } else {}; +in + +rec { + shell = "@bash@"; + + path = "@coreutils@"; + + system = "@system@"; + + shared = builtins.getEnv "_NIX_TEST_SHARED"; + + mkDerivation = args: + derivation ({ + inherit system; + builder = shell; + args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + PATH = path; + } // caArgs // removeAttrs args ["builder" "meta"]) + // { meta = args.meta or {}; }; +} diff -Nru nix-2.8.0/tests/dyn-drv/recursive-mod-json.nix nix-2.16.1+dfsg/tests/dyn-drv/recursive-mod-json.nix --- nix-2.8.0/tests/dyn-drv/recursive-mod-json.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/dyn-drv/recursive-mod-json.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,33 @@ +with import ./config.nix; + +let innerName = "foo"; in + +mkDerivation rec { + name = "${innerName}.drv"; + SHELL = shell; + + requiredSystemFeatures = [ "recursive-nix" ]; + + drv = builtins.unsafeDiscardOutputDependency (import ./text-hashed-output.nix).hello.drvPath; + + buildCommand = '' + export NIX_CONFIG='experimental-features = nix-command ca-derivations' + + PATH=${builtins.getEnv "EXTRA_PATH"}:$PATH + + # JSON of pre-existing drv + nix derivation show $drv | jq .[] > drv0.json + + # Fix name + jq < drv0.json '.name = "${innerName}"' > drv1.json + + # Extend `buildCommand` + jq < drv1.json '.env.buildCommand += "echo \"I am alive!\" >> $out/hello\n"' > drv0.json + + # Used as our output + cp $(nix derivation add < drv0.json) $out + ''; + __contentAddressed = true; + outputHashMode = "text"; + outputHashAlgo = "sha256"; +} diff -Nru nix-2.8.0/tests/dyn-drv/recursive-mod-json.sh nix-2.16.1+dfsg/tests/dyn-drv/recursive-mod-json.sh --- nix-2.8.0/tests/dyn-drv/recursive-mod-json.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/dyn-drv/recursive-mod-json.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,25 @@ +source common.sh + +# FIXME +if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi + +enableFeatures 'recursive-nix' +restartDaemon + +clearStore + +rm -f $TEST_ROOT/result + +EXTRA_PATH=$(dirname $(type -p nix)):$(dirname $(type -p jq)) +export EXTRA_PATH + +# Will produce a drv +metaDrv=$(nix-instantiate ./recursive-mod-json.nix) + +# computed "dynamic" derivation +drv=$(nix-store -r $metaDrv) + +# build that dyn drv +res=$(nix-store -r $drv) + +grep 'I am alive!' $res/hello diff -Nru nix-2.8.0/tests/dyn-drv/text-hashed-output.nix nix-2.16.1+dfsg/tests/dyn-drv/text-hashed-output.nix --- nix-2.8.0/tests/dyn-drv/text-hashed-output.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/dyn-drv/text-hashed-output.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,29 @@ +with import ./config.nix; + +# A simple content-addressed derivation. +# The derivation can be arbitrarily modified by passing a different `seed`, +# but the output will always be the same +rec { + hello = mkDerivation { + name = "hello"; + buildCommand = '' + set -x + echo "Building a CA derivation" + mkdir -p $out + echo "Hello World" > $out/hello + ''; + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + }; + producingDrv = mkDerivation { + name = "hello.drv"; + buildCommand = '' + echo "Copying the derivation" + cp ${builtins.unsafeDiscardOutputDependency hello.drvPath} $out + ''; + __contentAddressed = true; + outputHashMode = "text"; + outputHashAlgo = "sha256"; + }; +} diff -Nru nix-2.8.0/tests/dyn-drv/text-hashed-output.sh nix-2.16.1+dfsg/tests/dyn-drv/text-hashed-output.sh --- nix-2.8.0/tests/dyn-drv/text-hashed-output.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/dyn-drv/text-hashed-output.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +source common.sh + +# In the corresponding nix file, we have two derivations: the first, named root, +# is a normal recursive derivation, while the second, named dependent, has the +# new outputHashMode "text". Note that in "dependent", we don't refer to the +# build output of root, but only to the path of the drv file. For this reason, +# we only need to: +# +# - instantiate the root derivation +# - build the dependent derivation +# - check that the path of the output coincides with that of the original derivation + +drv=$(nix-instantiate ./text-hashed-output.nix -A hello) +nix show-derivation "$drv" + +drvProducingDrv=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) +nix show-derivation "$drvProducingDrv" + +out1=$(nix-build ./text-hashed-output.nix -A producingDrv --no-out-link) + +nix path-info $drv --derivation --json | jq +nix path-info $out1 --derivation --json | jq + +test $out1 == $drv diff -Nru nix-2.8.0/tests/eval.sh nix-2.16.1+dfsg/tests/eval.sh --- nix-2.8.0/tests/eval.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/eval.sh 2023-06-06 10:55:23.000000000 +0000 @@ -16,14 +16,28 @@ [[ $(nix eval int -f "./eval.nix") == 123 ]] [[ $(nix eval str -f "./eval.nix") == '"foo"' ]] [[ $(nix eval str --raw -f "./eval.nix") == 'foo' ]] -[[ $(nix eval attr -f "./eval.nix") == '{ foo = "bar"; }' ]] +[[ "$(nix eval attr -f "./eval.nix")" == '{ foo = "bar"; }' ]] [[ $(nix eval attr --json -f "./eval.nix") == '{"foo":"bar"}' ]] [[ $(nix eval int -f - < "./eval.nix") == 123 ]] +[[ "$(nix eval --expr '{"assert"=1;bar=2;}')" == '{ "assert" = 1; bar = 2; }' ]] +# Check if toFile can be utilized during restricted eval +[[ $(nix eval --restrict-eval --expr 'import (builtins.toFile "source" "42")') == 42 ]] nix-instantiate --eval -E 'assert 1 + 2 == 3; true' [[ $(nix-instantiate -A int --eval "./eval.nix") == 123 ]] [[ $(nix-instantiate -A str --eval "./eval.nix") == '"foo"' ]] -[[ $(nix-instantiate -A attr --eval "./eval.nix") == '{ foo = "bar"; }' ]] +[[ "$(nix-instantiate -A attr --eval "./eval.nix")" == '{ foo = "bar"; }' ]] [[ $(nix-instantiate -A attr --eval --json "./eval.nix") == '{"foo":"bar"}' ]] [[ $(nix-instantiate -A int --eval - < "./eval.nix") == 123 ]] +[[ "$(nix-instantiate --eval -E '{"assert"=1;bar=2;}')" == '{ "assert" = 1; bar = 2; }' ]] + +# Check that symlink cycles don't cause a hang. +ln -sfn cycle.nix $TEST_ROOT/cycle.nix +(! nix eval --file $TEST_ROOT/cycle.nix) + +# Check that relative symlinks are resolved correctly. +mkdir -p $TEST_ROOT/xyzzy $TEST_ROOT/foo +ln -sfn ../xyzzy $TEST_ROOT/foo/bar +printf 123 > $TEST_ROOT/xyzzy/default.nix +[[ $(nix eval --impure --expr "import $TEST_ROOT/foo/bar") = 123 ]] diff -Nru nix-2.8.0/tests/eval-store.sh nix-2.16.1+dfsg/tests/eval-store.sh --- nix-2.8.0/tests/eval-store.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/eval-store.sh 2023-06-06 10:55:23.000000000 +0000 @@ -2,7 +2,7 @@ # Using `--eval-store` with the daemon will eventually copy everything # to the build store, invalidating most of the tests here -needLocalStore +needLocalStore "“--eval-store” doesn't achieve much with the daemon" eval_store=$TEST_ROOT/eval-store diff -Nru nix-2.8.0/tests/experimental-features.sh nix-2.16.1+dfsg/tests/experimental-features.sh --- nix-2.8.0/tests/experimental-features.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/experimental-features.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,86 @@ +source common.sh + +# Skipping these two for now, because we actually *do* want flags and +# config settings to always show up in the manual, just be marked +# experimental. Will reenable once the manual generation takes advantage +# of the JSON metadata on this. +# +# # Without flakes, flake options should not show up +# # With flakes, flake options should show up +# +# function grep_both_ways { +# nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake +# nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake +# +# # Also, the order should not matter +# nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake +# nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake +# } +# +# # Simple case, the configuration effects the running command +# grep_both_ways show-config +# +# # Medium case, the configuration effects --help +# grep_both_ways store gc --help + +# Test settings that are gated on experimental features; the setting is ignored +# with a warning if the experimental feature is not enabled. The order of the +# `setting = value` lines in the configuration should not matter. + +# 'flakes' experimental-feature is disabled before, ignore and warn +NIX_CONFIG=' + experimental-features = nix-command + accept-flake-config = true +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "false" $TEST_ROOT/stdout +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr + +# 'flakes' experimental-feature is disabled after, ignore and warn +NIX_CONFIG=' + accept-flake-config = true + experimental-features = nix-command +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "false" $TEST_ROOT/stdout +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr + +# 'flakes' experimental-feature is enabled before, process +NIX_CONFIG=' + experimental-features = nix-command flakes + accept-flake-config = true +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "true" $TEST_ROOT/stdout +grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr + +# 'flakes' experimental-feature is enabled after, process +NIX_CONFIG=' + accept-flake-config = true + experimental-features = nix-command flakes +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "true" $TEST_ROOT/stdout +grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr + +function exit_code_both_ways { + expect 1 nix --experimental-features 'nix-command' "$@" 1>/dev/null + nix --experimental-features 'nix-command flakes' "$@" 1>/dev/null + + # Also, the order should not matter + expect 1 nix "$@" --experimental-features 'nix-command' 1>/dev/null + nix "$@" --experimental-features 'nix-command flakes' 1>/dev/null +} + +exit_code_both_ways show-config --flake-registry 'https://no' + +# Double check these are stable +nix --experimental-features '' --help 1>/dev/null +nix --experimental-features '' doctor --help 1>/dev/null +nix --experimental-features '' repl --help 1>/dev/null +nix --experimental-features '' upgrade-nix --help 1>/dev/null + +# These 3 arguments are currently given to all commands, which is wrong (as not +# all care). To deal with fixing later, we simply make them require the +# nix-command experimental features --- it so happens that the commands we wish +# stabilizing to do not need them anyways. +for arg in '--print-build-logs' '--offline' '--refresh'; do + nix --experimental-features 'nix-command' "$arg" --help 1>/dev/null + expect 1 nix --experimental-features '' "$arg" --help 1>/dev/null +done diff -Nru nix-2.8.0/tests/export-graph.sh nix-2.16.1+dfsg/tests/export-graph.sh --- nix-2.8.0/tests/export-graph.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/export-graph.sh 2023-06-06 10:55:23.000000000 +0000 @@ -4,7 +4,7 @@ clearProfiles checkRef() { - nix-store -q --references $TEST_ROOT/result | grep -q "$1" || fail "missing reference $1" + nix-store -q --references $TEST_ROOT/result | grepQuiet "$1"'$' || fail "missing reference $1" } # Test the export of the runtime dependency graph. diff -Nru nix-2.8.0/tests/fetchClosure.sh nix-2.16.1+dfsg/tests/fetchClosure.sh --- nix-2.8.0/tests/fetchClosure.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/fetchClosure.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,13 +1,12 @@ source common.sh enableFeatures "fetch-closure" -needLocalStore "'--no-require-sigs' can’t be used with the daemon" clearStore clearCacheCache # Initialize binary cache. -nonCaPath=$(nix build --json --file ./dependencies.nix | jq -r .[].outputs.out) +nonCaPath=$(nix build --json --file ./dependencies.nix --no-link | jq -r .[].outputs.out) caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]') nix copy --to file://$cacheDir $nonCaPath @@ -28,24 +27,28 @@ [ ! -e $nonCaPath ] [ -e $caPath ] -# In impure mode, we can use non-CA paths. -[[ $(nix eval --raw --no-require-sigs --impure --expr " - builtins.fetchClosure { - fromStore = \"file://$cacheDir\"; - fromPath = $nonCaPath; - } -") = $nonCaPath ]] +if [[ "$NIX_REMOTE" != "daemon" ]]; then + + # In impure mode, we can use non-CA paths. + [[ $(nix eval --raw --no-require-sigs --impure --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + } + ") = $nonCaPath ]] + + [ -e $nonCaPath ] -[ -e $nonCaPath ] +fi # 'toPath' set to empty string should fail but print the expected path. -nix eval -v --json --expr " +expectStderr 1 nix eval -v --json --expr " builtins.fetchClosure { fromStore = \"file://$cacheDir\"; fromPath = $nonCaPath; toPath = \"\"; } -" 2>&1 | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath" +" | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath" # If fromPath is CA, then toPath isn't needed. nix copy --to file://$cacheDir $caPath diff -Nru nix-2.8.0/tests/fetchGitRefs.sh nix-2.16.1+dfsg/tests/fetchGitRefs.sh --- nix-2.8.0/tests/fetchGitRefs.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/fetchGitRefs.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,9 +1,6 @@ source common.sh -if [[ -z $(type -p git) ]]; then - echo "Git not installed; skipping Git tests" - exit 99 -fi +requireGit clearStore @@ -56,7 +53,7 @@ else (! git check-ref-format --branch "$1" >/dev/null 2>&1) fi - nix --debug eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'invalid Git branch/tag name' >/dev/null + expect 1 nix --debug eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'invalid Git branch/tag name' >/dev/null } diff -Nru nix-2.8.0/tests/fetchGit.sh nix-2.16.1+dfsg/tests/fetchGit.sh --- nix-2.8.0/tests/fetchGit.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/fetchGit.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,9 +1,6 @@ source common.sh -if [[ -z $(type -p git) ]]; then - echo "Git not installed; skipping Git tests" - exit 99 -fi +requireGit clearStore @@ -24,12 +21,14 @@ git -C $repo add hello .gitignore git -C $repo commit -m 'Bla1' rev1=$(git -C $repo rev-parse HEAD) +git -C $repo tag -a tag1 -m tag1 echo world > $repo/hello git -C $repo commit -m 'Bla2' -a git -C $repo worktree add $TEST_ROOT/worktree echo hello >> $TEST_ROOT/worktree/hello rev2=$(git -C $repo rev-parse HEAD) +git -C $repo tag -a tag2 -m tag2 # Fetch a worktree unset _NIX_FORCE_HTTP @@ -120,6 +119,7 @@ path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $path2 = $path4 ]] +status=0 nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$? [[ "$status" = "102" ]] @@ -161,6 +161,14 @@ [[ $(cat $path4/hello) = dev ]] [[ $path3 = $path4 ]] +# Using remote path with branch other than 'master' should fetch the HEAD revision. +# (--tarball-ttl 0 to prevent using the cached repo above) +export _NIX_FORCE_HTTP=1 +path4=$(nix eval --tarball-ttl 0 --impure --raw --expr "(builtins.fetchGit $repo).outPath") +[[ $(cat $path4/hello) = dev ]] +[[ $path3 = $path4 ]] +unset _NIX_FORCE_HTTP + # Confirm same as 'dev' branch path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] @@ -209,6 +217,16 @@ path9=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; name = \"foo\"; }).outPath") [[ $path9 =~ -foo$ ]] +# Specifying a ref without a rev shouldn't pick a cached rev for a different ref +export _NIX_FORCE_HTTP=1 +rev_tag1_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/tags/tag1\"; }).rev") +rev_tag1=$(git -C $repo rev-parse refs/tags/tag1) +[[ $rev_tag1_nix = $rev_tag1 ]] +rev_tag2_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/tags/tag2\"; }).rev") +rev_tag2=$(git -C $repo rev-parse refs/tags/tag2) +[[ $rev_tag2_nix = $rev_tag2 ]] +unset _NIX_FORCE_HTTP + # should fail if there is no repo rm -rf $repo/.git (! nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") @@ -216,3 +234,17 @@ # should succeed for a repo without commits git init $repo path10=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") + +# should succeed for a path with a space +# regression test for #7707 +repo="$TEST_ROOT/a b" +git init "$repo" +git -C "$repo" config user.email "foobar@example.com" +git -C "$repo" config user.name "Foobar" + +echo utrecht > "$repo/hello" +touch "$repo/.gitignore" +git -C "$repo" add hello .gitignore +git -C "$repo" commit -m 'Bla1' +cd "$repo" +path11=$(nix eval --impure --raw --expr "(builtins.fetchGit ./.).outPath") diff -Nru nix-2.8.0/tests/fetchGitSubmodules.sh nix-2.16.1+dfsg/tests/fetchGitSubmodules.sh --- nix-2.8.0/tests/fetchGitSubmodules.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/fetchGitSubmodules.sh 2023-06-06 10:55:23.000000000 +0000 @@ -2,10 +2,7 @@ set -u -if [[ -z $(type -p git) ]]; then - echo "Git not installed; skipping Git submodule tests" - exit 99 -fi +requireGit clearStore @@ -14,6 +11,15 @@ rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix +# Submodules can't be fetched locally by default, which can cause +# information leakage vulnerabilities, but for these tests our +# submodule is intentionally local and it's all trusted, so we +# disable this restriction. Setting it per repo is not sufficient, as +# the repo-local config does not apply to the commands run from +# outside the repos by Nix. +export XDG_CONFIG_HOME=$TEST_HOME/.config +git config --global protocol.file.allow always + initGitRepo() { git init $1 git -C $1 config user.email "foobar@example.com" @@ -95,3 +101,28 @@ noSubmoduleRepo=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") [[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] + +# Test relative submodule URLs. +rm $TEST_HOME/.cache/nix/fetcher-cache* +rm -rf $rootRepo/.git $rootRepo/.gitmodules $rootRepo/sub +initGitRepo $rootRepo +git -C $rootRepo submodule add ../gitSubmodulesSub sub +git -C $rootRepo commit -m "Add submodule" +rev2=$(git -C $rootRepo rev-parse HEAD) +pathWithRelative=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev2\"; submodules = true; }).outPath") +diff -r -x .gitmodules $pathWithSubmodules $pathWithRelative + +# Test clones that have an upstream with relative submodule URLs. +rm $TEST_HOME/.cache/nix/fetcher-cache* +cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path not work relative to $cloneRepo +git clone $rootRepo $cloneRepo +pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") +[[ $pathIndirect = $pathWithRelative ]] + +# Test that if the clone has the submodule already, we're not fetching +# it again. +git -C $cloneRepo submodule update --init +rm $TEST_HOME/.cache/nix/fetcher-cache* +rm -rf $subRepo +pathSubmoduleGone=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") +[[ $pathSubmoduleGone = $pathWithRelative ]] diff -Nru nix-2.8.0/tests/fetchMercurial.sh nix-2.16.1+dfsg/tests/fetchMercurial.sh --- nix-2.8.0/tests/fetchMercurial.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/fetchMercurial.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,9 +1,6 @@ source common.sh -if [[ -z $(type -p hg) ]]; then - echo "Mercurial not installed; skipping Mercurial tests" - exit 99 -fi +[[ $(type -p hq) ]] || skipTest "Mercurial not installed" clearStore diff -Nru nix-2.8.0/tests/fetchTree-file.sh nix-2.16.1+dfsg/tests/fetchTree-file.sh --- nix-2.8.0/tests/fetchTree-file.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/fetchTree-file.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,105 @@ +source common.sh + +clearStore + +cd "$TEST_ROOT" + +test_fetch_file () { + echo foo > test_input + + input_hash="$(nix hash path test_input)" + + nix eval --impure --file - < inputs/test_input_file + tar cfa test_input.tar.gz inputs + cp test_input.tar.gz test_input_no_ext + input_tarball_hash="$(nix hash path test_input.tar.gz)" + input_directory_hash="$(nix hash path inputs)" + + cat < flake.nix + { + inputs.no_ext_default_no_unpack = { + url = "file://$PWD/test_input_no_ext"; + flake = false; + }; + inputs.no_ext_explicit_unpack = { + url = "tarball+file://$PWD/test_input_no_ext"; + flake = false; + }; + inputs.tarball_default_unpack = { + url = "file://$PWD/test_input.tar.gz"; + flake = false; + }; + inputs.tarball_explicit_no_unpack = { + url = "file+file://$PWD/test_input.tar.gz"; + flake = false; + }; + outputs = { ... }: {}; + } +EOF + + nix flake update + nix eval --file - < flake.nix + { + inputs.tarball = { + url = "file://$PWD/test_input.tar.gz"; + flake = false; + }; + outputs = { self, tarball }: { + foo = builtins.readFile "\${tarball}/test_input_file"; + }; + } + nix flake update + + clearStore + "$NIX_DAEMON_PACKAGE/bin/nix" eval .#foo +EOF +} + +test_fetch_file +test_file_flake_input diff -Nru nix-2.8.0/tests/fetchurl.sh nix-2.16.1+dfsg/tests/fetchurl.sh --- nix-2.8.0/tests/fetchurl.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/fetchurl.sh 2023-06-06 10:55:23.000000000 +0000 @@ -62,7 +62,7 @@ outPath=$(nix-build -vvvvv --expr 'import ' --argstr url file://$nar --argstr sha256 $hash \ --arg unpack true --argstr name xyzzy --no-out-link) -echo $outPath | grep -q 'xyzzy' +echo $outPath | grepQuiet 'xyzzy' test -x $outPath/fetchurl.sh test -L $outPath/symlink diff -Nru nix-2.8.0/tests/flake-bundler.sh nix-2.16.1+dfsg/tests/flake-bundler.sh --- nix-2.8.0/tests/flake-bundler.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flake-bundler.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,37 +0,0 @@ -source common.sh - -clearStore -rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local - -cp ./simple.nix ./simple.builder.sh ./config.nix $TEST_HOME - -cd $TEST_HOME - -cat < flake.nix -{ - outputs = {self}: { - bundlers.$system = rec { - simple = drv: - if drv?type && drv.type == "derivation" - then drv - else self.packages.$system.default; - default = simple; - }; - packages.$system.default = import ./simple.nix; - apps.$system.default = { - type = "app"; - program = "\${import ./simple.nix}/hello"; - }; - }; -} -EOF -nix build .# -nix bundle --bundler .# .# -nix bundle --bundler .#bundlers.$system.default .#packages.$system.default -nix bundle --bundler .#bundlers.$system.simple .#packages.$system.default - -nix bundle --bundler .#bundlers.$system.default .#apps.$system.default -nix bundle --bundler .#bundlers.$system.simple .#apps.$system.default - -clearStore - diff -Nru nix-2.8.0/tests/flake-local-settings.sh nix-2.16.1+dfsg/tests/flake-local-settings.sh --- nix-2.8.0/tests/flake-local-settings.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flake-local-settings.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,45 +0,0 @@ -source common.sh - -clearStore -rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local - -cp ./simple.nix ./simple.builder.sh ./config.nix $TEST_HOME - -cd $TEST_HOME - -rm -f post-hook-ran -cat < echoing-post-hook.sh -#!/bin/sh - -echo "ThePostHookRan as \$0" > $PWD/post-hook-ran -EOF -chmod +x echoing-post-hook.sh - -cat < flake.nix -{ - nixConfig.post-build-hook = ./echoing-post-hook.sh; - nixConfig.allow-dirty = false; # See #5621 - - outputs = a: { - packages.$system.default = import ./simple.nix; - }; -} -EOF - -# Without --accept-flake-config, the post hook should not run. -nix build < /dev/null -(! [[ -f post-hook-ran ]]) -clearStore - -nix build --accept-flake-config -test -f post-hook-ran || fail "The post hook should have ran" - -# Make sure that the path to the post hook doesn’t change if we change -# something in the flake. -# Otherwise the user would have to re-validate the setting each time. -mv post-hook-ran previous-post-hook-run -echo "# Dummy comment" >> flake.nix -clearStore -nix build --accept-flake-config -diff -q post-hook-ran previous-post-hook-run || \ - fail "Both post hook runs should report the same filename" diff -Nru nix-2.8.0/tests/flakes/absolute-paths.sh nix-2.16.1+dfsg/tests/flakes/absolute-paths.sh --- nix-2.8.0/tests/flakes/absolute-paths.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes/absolute-paths.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,17 @@ +source ./common.sh + +requireGit + +flake1Dir=$TEST_ROOT/flake1 +flake2Dir=$TEST_ROOT/flake2 + +createGitRepo $flake1Dir +cat > $flake1Dir/flake.nix < $flake1Dir/flake.nix < $flake1Dir/foo + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a1 +[[ -e $TEST_ROOT/result/simple.nix ]] + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a2 +[[ $(cat $TEST_ROOT/result) = bar ]] + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a3 + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a4 + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a6 +[[ -e $TEST_ROOT/result/simple.nix ]] + +nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a8 +diff common.sh $TEST_ROOT/result + +expectStderr 1 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a9 \ + | grepQuiet "has 0 entries in its context. It should only have exactly one entry" + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a10 +[[ $(readlink -e $TEST_ROOT/result) = *simple.drv ]] + +expectStderr 1 nix build --json --out-link $TEST_ROOT/result $flake1Dir#a11 \ + | grepQuiet "has a context which refers to a complete source and binary closure" + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a12 +[[ -e $TEST_ROOT/result/hello ]] + +expectStderr 1 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a13 \ + | grepQuiet "has 2 entries in its context. It should only have exactly one entry" diff -Nru nix-2.8.0/tests/flakes/bundle.sh nix-2.16.1+dfsg/tests/flakes/bundle.sh --- nix-2.8.0/tests/flakes/bundle.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes/bundle.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,32 @@ +source common.sh + +cp ../simple.nix ../simple.builder.sh ../config.nix $TEST_HOME + +cd $TEST_HOME + +cat < flake.nix +{ + outputs = {self}: { + bundlers.$system = rec { + simple = drv: + if drv?type && drv.type == "derivation" + then drv + else self.packages.$system.default; + default = simple; + }; + packages.$system.default = import ./simple.nix; + apps.$system.default = { + type = "app"; + program = "\${import ./simple.nix}/hello"; + }; + }; +} +EOF + +nix build .# +nix bundle --bundler .# .# +nix bundle --bundler .#bundlers.$system.default .#packages.$system.default +nix bundle --bundler .#bundlers.$system.simple .#packages.$system.default + +nix bundle --bundler .#bundlers.$system.default .#apps.$system.default +nix bundle --bundler .#bundlers.$system.simple .#apps.$system.default diff -Nru nix-2.8.0/tests/flakes/check.sh nix-2.16.1+dfsg/tests/flakes/check.sh --- nix-2.8.0/tests/flakes/check.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes/check.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,79 @@ +source common.sh + +flakeDir=$TEST_ROOT/flake3 +mkdir -p $flakeDir + +cat > $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +echo "$checkRes" | grepQuiet "packages.system-1.default" +echo "$checkRes" | grepQuiet "packages.system-2.default" diff -Nru nix-2.8.0/tests/flakes/circular.sh nix-2.16.1+dfsg/tests/flakes/circular.sh --- nix-2.8.0/tests/flakes/circular.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes/circular.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,49 @@ +# Test circular flake dependencies. +source ./common.sh + +requireGit + +flakeA=$TEST_ROOT/flakeA +flakeB=$TEST_ROOT/flakeB + +createGitRepo $flakeA +createGitRepo $flakeB + +cat > $flakeA/flake.nix < $flakeB/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < echoing-post-hook.sh +#!/bin/sh + +echo "ThePostHookRan as \$0" > $PWD/post-hook-ran +EOF +chmod +x echoing-post-hook.sh + +cat < flake.nix +{ + nixConfig.post-build-hook = ./echoing-post-hook.sh; + nixConfig.allow-dirty = false; # See #5621 + + outputs = a: { + packages.$system.default = import ./simple.nix; + }; +} +EOF + +# Without --accept-flake-config, the post hook should not run. +nix build < /dev/null +(! [[ -f post-hook-ran ]]) +clearStore + +nix build --accept-flake-config +test -f post-hook-ran || fail "The post hook should have ran" + +# Make sure that the path to the post hook doesn’t change if we change +# something in the flake. +# Otherwise the user would have to re-validate the setting each time. +mv post-hook-ran previous-post-hook-run +echo "# Dummy comment" >> flake.nix +clearStore +nix build --accept-flake-config +diff -q post-hook-ran previous-post-hook-run || \ + fail "Both post hook runs should report the same filename" diff -Nru nix-2.8.0/tests/flakes/flake-in-submodule.sh nix-2.16.1+dfsg/tests/flakes/flake-in-submodule.sh --- nix-2.8.0/tests/flakes/flake-in-submodule.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes/flake-in-submodule.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,52 @@ +source common.sh + +# Tests that: +# - flake.nix may reside inside of a git submodule +# - the flake can access content outside of the submodule +# +# rootRepo +# ├── root.nix +# └── submodule +# ├── flake.nix +# └── sub.nix + + +requireGit + +clearStore + +# Submodules can't be fetched locally by default. +# See fetchGitSubmodules.sh +export XDG_CONFIG_HOME=$TEST_HOME/.config +git config --global protocol.file.allow always + + +rootRepo=$TEST_ROOT/rootRepo +subRepo=$TEST_ROOT/submodule + + +createGitRepo $subRepo +cat > $subRepo/flake.nix < $subRepo/sub.nix +git -C $subRepo add flake.nix sub.nix +git -C $subRepo commit -m Initial + +createGitRepo $rootRepo + +git -C $rootRepo submodule init +git -C $rootRepo submodule add $subRepo submodule +echo '"expression in root repo"' > $rootRepo/root.nix +git -C $rootRepo add root.nix +git -C $rootRepo commit -m "Add root.nix" + +# Flake can live inside a submodule and can be accessed via ?dir=submodule +[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]] +# The flake can access content outside of the submodule +[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]] diff -Nru nix-2.8.0/tests/flakes/flakes.sh nix-2.16.1+dfsg/tests/flakes/flakes.sh --- nix-2.8.0/tests/flakes/flakes.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes/flakes.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,506 @@ +source ./common.sh + +requireGit + +clearStore +rm -rf $TEST_HOME/.cache $TEST_HOME/.config + +flake1Dir=$TEST_ROOT/flake1 +flake2Dir=$TEST_ROOT/flake2 +flake3Dir=$TEST_ROOT/flake3 +flake5Dir=$TEST_ROOT/flake5 +flake7Dir=$TEST_ROOT/flake7 +nonFlakeDir=$TEST_ROOT/nonFlake +badFlakeDir=$TEST_ROOT/badFlake +flakeGitBare=$TEST_ROOT/flakeGitBare + +for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $nonFlakeDir; do + # Give one repo a non-main initial branch. + extraArgs= + if [[ $repo == $flake2Dir ]]; then + extraArgs="--initial-branch=main" + fi + + createGitRepo "$repo" "$extraArgs" +done + +createSimpleGitFlake $flake1Dir + +cat > $flake2Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/default.nix < $nonFlakeDir/README.md <> $flake1Dir/flake.nix +flake1OriginalCommit=$(git -C $flake1Dir rev-parse HEAD) +git -C $flake1Dir commit -a -m 'Foo' +flake1NewCommit=$(git -C $flake1Dir rev-parse HEAD) +hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) +[[ $hash1 != $hash2 ]] + +# Test 'nix build' on a flake. +nix build -o $TEST_ROOT/result flake1#foo +[[ -e $TEST_ROOT/result/hello ]] + +# Test packages.default. +nix build -o $TEST_ROOT/result flake1 +[[ -e $TEST_ROOT/result/hello ]] + +nix build -o $TEST_ROOT/result $flake1Dir +nix build -o $TEST_ROOT/result git+file://$flake1Dir + +# Check that store symlinks inside a flake are not interpreted as flakes. +nix build -o $flake1Dir/result git+file://$flake1Dir +nix path-info $flake1Dir/result + +# 'getFlake' on an unlocked flakeref should fail in pure mode, but +# succeed in impure mode. +(! nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") +nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure + +# 'getFlake' on a locked flakeref should succeed even in pure mode. +nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" + +# Building a flake with an unlocked dependency should fail in pure mode. +(! nix build -o $TEST_ROOT/result flake2#bar --no-registries) +(! nix build -o $TEST_ROOT/result flake2#bar --no-use-registries) +(! nix eval --expr "builtins.getFlake \"$flake2Dir\"") + +# But should succeed in impure mode. +(! nix build -o $TEST_ROOT/result flake2#bar --impure) +nix build -o $TEST_ROOT/result flake2#bar --impure --no-write-lock-file +nix eval --expr "builtins.getFlake \"$flake2Dir\"" --impure + +# Building a local flake with an unlocked dependency should fail with --no-update-lock-file. +expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' + +# But it should succeed without that flag. +nix build -o $TEST_ROOT/result $flake2Dir#bar --no-write-lock-file +expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' +nix build -o $TEST_ROOT/result $flake2Dir#bar --commit-lock-file +[[ -e $flake2Dir/flake.lock ]] +[[ -z $(git -C $flake2Dir diff main || echo failed) ]] + +# Rerunning the build should not change the lockfile. +nix build -o $TEST_ROOT/result $flake2Dir#bar +[[ -z $(git -C $flake2Dir diff main || echo failed) ]] + +# Building with a lockfile should not require a fetch of the registry. +nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh +nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh +nix build -o $TEST_ROOT/result --no-use-registries $flake2Dir#bar --refresh + +# Updating the flake should not change the lockfile. +nix flake lock $flake2Dir +[[ -z $(git -C $flake2Dir diff main || echo failed) ]] + +# Now we should be able to build the flake in pure mode. +nix build -o $TEST_ROOT/result flake2#bar + +# Or without a registry. +nix build -o $TEST_ROOT/result --no-registries git+file://$flake2Dir#bar --refresh +nix build -o $TEST_ROOT/result --no-use-registries git+file://$flake2Dir#bar --refresh + +# Test whether indirect dependencies work. +nix build -o $TEST_ROOT/result $flake3Dir#xyzzy +git -C $flake3Dir add flake.lock + +# Add dependency to flake3. +rm $flake3Dir/flake.nix + +cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < \$out + [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]] + [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]] + ''; + }; + }; +} +EOF + +cp ../config.nix $flake3Dir + +git -C $flake3Dir add flake.nix config.nix +git -C $flake3Dir commit -m 'Add nonFlakeInputs' + +# Check whether `nix build` works with a lockfile which is missing a +# nonFlakeInputs. +nix build -o $TEST_ROOT/result $flake3Dir#sth --commit-lock-file + +nix build -o $TEST_ROOT/result flake3#fnord +[[ $(cat $TEST_ROOT/result) = FNORD ]] + +# Check whether flake input fetching is lazy: flake3#sth does not +# depend on flake2, so this shouldn't fail. +rm -rf $TEST_HOME/.cache +clearStore +mv $flake2Dir $flake2Dir.tmp +mv $nonFlakeDir $nonFlakeDir.tmp +nix build -o $TEST_ROOT/result flake3#sth +(! nix build -o $TEST_ROOT/result flake3#xyzzy) +(! nix build -o $TEST_ROOT/result flake3#fnord) +mv $flake2Dir.tmp $flake2Dir +mv $nonFlakeDir.tmp $nonFlakeDir +nix build -o $TEST_ROOT/result flake3#xyzzy flake3#fnord + +# Test doing multiple `lookupFlake`s +nix build -o $TEST_ROOT/result flake4#xyzzy + +# Test 'nix flake update' and --override-flake. +nix flake lock $flake3Dir +[[ -z $(git -C $flake3Dir diff master || echo failed) ]] + +nix flake update $flake3Dir --override-flake flake2 nixpkgs +[[ ! -z $(git -C $flake3Dir diff master || echo failed) ]] + +# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore +git -C $flake3Dir checkout -b removeXyzzy +rm $flake3Dir/flake.nix + +cat > $flake3Dir/flake.nix < \$out + ''; + }; + }; +} +EOF +nix flake lock $flake3Dir +git -C $flake3Dir add flake.nix flake.lock +git -C $flake3Dir commit -m 'Remove packages.xyzzy' +git -C $flake3Dir checkout master + +# Test whether fuzzy-matching works for registry entries. +(! nix build -o $TEST_ROOT/result flake4/removeXyzzy#xyzzy) +nix build -o $TEST_ROOT/result flake4/removeXyzzy#sth + +# Testing the nix CLI +nix registry add flake1 flake3 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry pin flake1 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry pin flake1 flake3 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry remove flake1 +[[ $(nix registry list | wc -l) == 5 ]] + +# Test 'nix registry list' with a disabled global registry. +nix registry add user-flake1 git+file://$flake1Dir +nix registry add user-flake2 git+file://$flake2Dir +[[ $(nix --flake-registry "" registry list | wc -l) == 2 ]] +nix --flake-registry "" registry list | grepQuietInverse '^global' # nothing in global registry +nix --flake-registry "" registry list | grepQuiet '^user' +nix registry remove user-flake1 +nix registry remove user-flake2 +[[ $(nix registry list | wc -l) == 5 ]] + +# Test 'nix flake clone'. +rm -rf $TEST_ROOT/flake1-v2 +nix flake clone flake1 --dest $TEST_ROOT/flake1-v2 +[ -e $TEST_ROOT/flake1-v2/flake.nix ] + +# Test 'follows' inputs. +cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $badFlakeDir/flake.nix +nix store delete $(nix store add-path $badFlakeDir) + +[[ $(nix path-info $(nix store add-path $flake1Dir)) =~ flake1 ]] +[[ $(nix path-info path:$(nix store add-path $flake1Dir)) =~ simple ]] + +# Test fetching flakerefs in the legacy CLI. +[[ $(nix-instantiate --eval flake:flake3 -A x) = 123 ]] +[[ $(nix-instantiate --eval flake:git+file://$flake3Dir -A x) = 123 ]] +[[ $(nix-instantiate -I flake3=flake:flake3 --eval '' -A x) = 123 ]] +[[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] + +# Test alternate lockfile paths. +nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2.lock +cmp $flake2Dir/flake.lock $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one + +nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit +expectStderr 1 cmp $flake2Dir/flake.lock $TEST_ROOT/flake2-overridden.lock +nix flake metadata $flake2Dir --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit + +# reference-lock-file can only be used if allow-dirty is set. +expectStderr 1 nix flake metadata $flake2Dir --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock diff -Nru nix-2.8.0/tests/flakes/follow-paths.sh nix-2.16.1+dfsg/tests/flakes/follow-paths.sh --- nix-2.8.0/tests/flakes/follow-paths.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes/follow-paths.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,150 @@ +source ./common.sh + +requireGit + +flakeFollowsA=$TEST_ROOT/follows/flakeA +flakeFollowsB=$TEST_ROOT/follows/flakeA/flakeB +flakeFollowsC=$TEST_ROOT/follows/flakeA/flakeB/flakeC +flakeFollowsD=$TEST_ROOT/follows/flakeA/flakeD +flakeFollowsE=$TEST_ROOT/follows/flakeA/flakeE + +# Test following path flakerefs. +createGitRepo $flakeFollowsA +mkdir -p $flakeFollowsB +mkdir -p $flakeFollowsC +mkdir -p $flakeFollowsD +mkdir -p $flakeFollowsE + +cat > $flakeFollowsA/flake.nix < $flakeFollowsB/flake.nix < $flakeFollowsC/flake.nix < $flakeFollowsD/flake.nix < $flakeFollowsE/flake.nix < $flakeFollowsA/flake.nix < $flakeFollowsA/flake.nix <&1 | grep 'points outside' + +# Non-existant follows should print a warning. +cat >$flakeFollowsA/flake.nix <&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" +nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" diff -Nru nix-2.8.0/tests/flakes/init.sh nix-2.16.1+dfsg/tests/flakes/init.sh --- nix-2.8.0/tests/flakes/init.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes/init.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,87 @@ +source ./common.sh + +requireGit + +templatesDir=$TEST_ROOT/templates +flakeDir=$TEST_ROOT/flake +nixpkgsDir=$TEST_ROOT/nixpkgs + +nix registry add --registry $registry templates git+file://$templatesDir +nix registry add --registry $registry nixpkgs git+file://$nixpkgsDir + +createGitRepo $nixpkgsDir +createSimpleGitFlake $nixpkgsDir + +# Test 'nix flake init'. +createGitRepo $templatesDir + +cat > $templatesDir/flake.nix < $templatesDir/trivial/flake.nix < $templatesDir/trivial/a +echo b > $templatesDir/trivial/b + +git -C $templatesDir add flake.nix trivial/ +git -C $templatesDir commit -m 'Initial' + +nix flake check templates +nix flake show templates +nix flake show templates --json | jq + +createGitRepo $flakeDir +(cd $flakeDir && nix flake init) +(cd $flakeDir && nix flake init) # check idempotence +git -C $flakeDir add flake.nix +nix flake check $flakeDir +nix flake show $flakeDir +nix flake show $flakeDir --json | jq +git -C $flakeDir commit -a -m 'Initial' + +# Test 'nix flake init' with benign conflicts +createGitRepo "$flakeDir" +echo a > $flakeDir/a +(cd $flakeDir && nix flake init) # check idempotence + +# Test 'nix flake init' with conflicts +createGitRepo "$flakeDir" +echo b > $flakeDir/a +pushd $flakeDir +(! nix flake init) |& grep "refusing to overwrite existing file '$flakeDir/a'" +popd +git -C $flakeDir commit -a -m 'Changed' + +# Test 'nix flake new'. +rm -rf $flakeDir +nix flake new -t templates#trivial $flakeDir +nix flake new -t templates#trivial $flakeDir # check idempotence +nix flake check $flakeDir diff -Nru nix-2.8.0/tests/flakes/inputs.sh nix-2.16.1+dfsg/tests/flakes/inputs.sh --- nix-2.8.0/tests/flakes/inputs.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes/inputs.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,80 @@ +source ./common.sh + +requireGit + + +test_subdir_self_path() { + baseDir=$TEST_ROOT/$RANDOM + flakeDir=$baseDir/b-low + mkdir -p $flakeDir + writeSimpleFlake $baseDir + writeSimpleFlake $flakeDir + + echo all good > $flakeDir/message + cat > $flakeDir/flake.nix < $flakeDir/message + cat > $flakeDir/flake.nix < $clientDir/flake.nix < flake.nix +{ + outputs = {self}: { + packages.$system.pkgAsPkg = (import ./shell-hello.nix).hello; + packages.$system.appAsApp = self.packages.$system.appAsApp; + + apps.$system.pkgAsApp = self.packages.$system.pkgAsPkg; + apps.$system.appAsApp = { + type = "app"; + program = "\${(import ./shell-hello.nix).hello}/bin/hello"; + }; + }; +} +EOF +nix run --no-write-lock-file .#appAsApp +nix run --no-write-lock-file .#pkgAsPkg + +! nix run --no-write-lock-file .#pkgAsApp || fail "'nix run' shouldn’t accept an 'app' defined under 'packages'" +! nix run --no-write-lock-file .#appAsPkg || fail "elements of 'apps' should be of type 'app'" + +clearStore + diff -Nru nix-2.8.0/tests/flakes/search-root.sh nix-2.16.1+dfsg/tests/flakes/search-root.sh --- nix-2.8.0/tests/flakes/search-root.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes/search-root.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,50 @@ +source common.sh + +clearStore + +writeSimpleFlake $TEST_HOME +cd $TEST_HOME +mkdir -p foo/subdir + +echo '{ outputs = _: {}; }' > foo/flake.nix +cat < flake.nix +{ + inputs.foo.url = "$PWD/foo"; + outputs = a: { + packages.$system = rec { + test = import ./simple.nix; + default = test; + }; + }; +} +EOF +mkdir subdir +pushd subdir + +success=("" . .# .#test ../subdir ../subdir#test "$PWD") +failure=("path:$PWD") + +for i in "${success[@]}"; do + nix build $i || fail "flake should be found by searching up directories" +done + +for i in "${failure[@]}"; do + ! nix build $i || fail "flake should not search up directories when using 'path:'" +done + +popd + +nix build --override-input foo . || fail "flake should search up directories when not an installable" + +sed "s,$PWD/foo,$PWD/foo/subdir,g" -i flake.nix +! nix build || fail "flake should not search upwards when part of inputs" + +if [[ -n $(type -p git) ]]; then + pushd subdir + git init + for i in "${success[@]}" "${failure[@]}"; do + ! nix build $i || fail "flake should not search past a git repository" + done + rm -rf .git + popd +fi diff -Nru nix-2.8.0/tests/flakes/show.sh nix-2.16.1+dfsg/tests/flakes/show.sh --- nix-2.8.0/tests/flakes/show.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes/show.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,87 @@ +source ./common.sh + +flakeDir=$TEST_ROOT/flake +mkdir -p "$flakeDir" + +writeSimpleFlake "$flakeDir" +cd "$flakeDir" + + +# By default: Only show the packages content for the current system and no +# legacyPackages at all +nix flake show --json > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.packages.someOtherSystem.default == {}; +assert show_output.packages.${builtins.currentSystem}.default.name == "simple"; +assert show_output.legacyPackages.${builtins.currentSystem} == {}; +true +' + +# With `--all-systems`, show the packages for all systems +nix flake show --json --all-systems > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.packages.someOtherSystem.default.name == "simple"; +assert show_output.legacyPackages.${builtins.currentSystem} == {}; +true +' + +# With `--legacy`, show the legacy packages +nix flake show --json --legacy > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.legacyPackages.${builtins.currentSystem}.hello.name == "simple"; +true +' + +# Test that attributes are only reported when they have actual content +cat >flake.nix < show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output == { }; +true +' + +# Test that attributes with errors are handled correctly. +# nixpkgs.legacyPackages is a particularly prominent instance of this. +cat >flake.nix < show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.legacyPackages.${builtins.currentSystem}.AAAAAASomeThingsFailToEvaluate == { }; +assert show_output.legacyPackages.${builtins.currentSystem}.simple.name == "simple"; +true +' diff -Nru nix-2.8.0/tests/flakes/unlocked-override.sh nix-2.16.1+dfsg/tests/flakes/unlocked-override.sh --- nix-2.8.0/tests/flakes/unlocked-override.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes/unlocked-override.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,30 @@ +source ./common.sh + +requireGit + +flake1Dir=$TEST_ROOT/flake1 +flake2Dir=$TEST_ROOT/flake2 + +createGitRepo $flake1Dir +cat > $flake1Dir/flake.nix < $flake1Dir/x.nix +git -C $flake1Dir add flake.nix x.nix +git -C $flake1Dir commit -m Initial + +createGitRepo $flake2Dir +cat > $flake2Dir/flake.nix < $flake1Dir/x.nix + +[[ $(nix eval --json $flake2Dir#x --override-input flake1 $TEST_ROOT/flake1) = 456 ]] diff -Nru nix-2.8.0/tests/flake-searching.sh nix-2.16.1+dfsg/tests/flake-searching.sh --- nix-2.8.0/tests/flake-searching.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flake-searching.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,52 +0,0 @@ -source common.sh - -if [[ -z $(type -p git) ]]; then - echo "Git not installed; skipping flake search tests" - exit 99 -fi - -clearStore - -cp ./simple.nix ./simple.builder.sh ./config.nix $TEST_HOME -cd $TEST_HOME -mkdir -p foo/subdir -echo '{ outputs = _: {}; }' > foo/flake.nix -cat < flake.nix -{ - inputs.foo.url = "$PWD/foo"; - outputs = a: { - packages.$system = rec { - test = import ./simple.nix; - default = test; - }; - }; -} -EOF -mkdir subdir -pushd subdir - -success=("" . .# .#test ../subdir ../subdir#test "$PWD") -failure=("path:$PWD") - -for i in "${success[@]}"; do - nix build $i || fail "flake should be found by searching up directories" -done - -for i in "${failure[@]}"; do - ! nix build $i || fail "flake should not search up directories when using 'path:'" -done - -popd - -nix build --override-input foo . || fail "flake should search up directories when not an installable" - -sed "s,$PWD/foo,$PWD/foo/subdir,g" -i flake.nix -! nix build || fail "flake should not search upwards when part of inputs" - -pushd subdir -git init -for i in "${success[@]}" "${failure[@]}"; do - ! nix build $i || fail "flake should not search past a git repository" -done -rm -rf .git -popd diff -Nru nix-2.8.0/tests/flakes-run.sh nix-2.16.1+dfsg/tests/flakes-run.sh --- nix-2.8.0/tests/flakes-run.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes-run.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,29 +0,0 @@ -source common.sh - -clearStore -rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local -cp ./shell-hello.nix ./config.nix $TEST_HOME -cd $TEST_HOME - -cat < flake.nix -{ - outputs = {self}: { - packages.$system.pkgAsPkg = (import ./shell-hello.nix).hello; - packages.$system.appAsApp = self.packages.$system.appAsApp; - - apps.$system.pkgAsApp = self.packages.$system.pkgAsPkg; - apps.$system.appAsApp = { - type = "app"; - program = "\${(import ./shell-hello.nix).hello}/bin/hello"; - }; - }; -} -EOF -nix run --no-write-lock-file .#appAsApp -nix run --no-write-lock-file .#pkgAsPkg - -! nix run --no-write-lock-file .#pkgAsApp || fail "'nix run' shouldn’t accept an 'app' defined under 'packages'" -! nix run --no-write-lock-file .#appAsPkg || fail "elements of 'apps' should be of type 'app'" - -clearStore - diff -Nru nix-2.8.0/tests/flakes.sh nix-2.16.1+dfsg/tests/flakes.sh --- nix-2.8.0/tests/flakes.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/flakes.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,839 +0,0 @@ -source common.sh - -if [[ -z $(type -p git) ]]; then - echo "Git not installed; skipping flake tests" - exit 99 -fi - -clearStore -rm -rf $TEST_HOME/.cache $TEST_HOME/.config - -registry=$TEST_ROOT/registry.json - -flake1Dir=$TEST_ROOT/flake1 -flake2Dir=$TEST_ROOT/flake2 -flake3Dir=$TEST_ROOT/flake3 -flake5Dir=$TEST_ROOT/flake5 -flake6Dir=$TEST_ROOT/flake6 -flake7Dir=$TEST_ROOT/flake7 -templatesDir=$TEST_ROOT/templates -nonFlakeDir=$TEST_ROOT/nonFlake -badFlakeDir=$TEST_ROOT/badFlake -flakeA=$TEST_ROOT/flakeA -flakeB=$TEST_ROOT/flakeB -flakeGitBare=$TEST_ROOT/flakeGitBare -flakeFollowsA=$TEST_ROOT/follows/flakeA -flakeFollowsB=$TEST_ROOT/follows/flakeA/flakeB -flakeFollowsC=$TEST_ROOT/follows/flakeA/flakeB/flakeC -flakeFollowsD=$TEST_ROOT/follows/flakeA/flakeD -flakeFollowsE=$TEST_ROOT/follows/flakeA/flakeE - -for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeDir $flakeA $flakeB $flakeFollowsA; do - rm -rf $repo $repo.tmp - mkdir -p $repo - git -C $repo init - git -C $repo config user.email "foobar@example.com" - git -C $repo config user.name "Foobar" -done - -cat > $flake1Dir/flake.nix < $flake2Dir/flake.nix < $flake3Dir/flake.nix < $nonFlakeDir/README.md <> $flake1Dir/flake.nix -git -C $flake1Dir commit -a -m 'Foo' -hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) -[[ $hash1 != $hash2 ]] - -# Test 'nix build' on a flake. -nix build -o $TEST_ROOT/result flake1#foo -[[ -e $TEST_ROOT/result/hello ]] - -# Test packages.default. -nix build -o $TEST_ROOT/result flake1 -[[ -e $TEST_ROOT/result/hello ]] - -nix build -o $TEST_ROOT/result $flake1Dir -nix build -o $TEST_ROOT/result git+file://$flake1Dir - -# Check that store symlinks inside a flake are not interpreted as flakes. -nix build -o $flake1Dir/result git+file://$flake1Dir -nix path-info $flake1Dir/result - -# 'getFlake' on a mutable flakeref should fail in pure mode, but succeed in impure mode. -(! nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure - -# 'getFlake' on an immutable flakeref should succeed even in pure mode. -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" - -# Building a flake with an unlocked dependency should fail in pure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --no-registries) -(! nix build -o $TEST_ROOT/result flake2#bar --no-use-registries) -(! nix eval --expr "builtins.getFlake \"$flake2Dir\"") - -# But should succeed in impure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --impure) -nix build -o $TEST_ROOT/result flake2#bar --impure --no-write-lock-file - -# Building a local flake with an unlocked dependency should fail with --no-update-lock-file. -nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' - -# But it should succeed without that flag. -nix build -o $TEST_ROOT/result $flake2Dir#bar --no-write-lock-file -nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' -nix build -o $TEST_ROOT/result $flake2Dir#bar --commit-lock-file -[[ -e $flake2Dir/flake.lock ]] -[[ -z $(git -C $flake2Dir diff master) ]] - -# Rerunning the build should not change the lockfile. -nix build -o $TEST_ROOT/result $flake2Dir#bar -[[ -z $(git -C $flake2Dir diff master) ]] - -# Building with a lockfile should not require a fetch of the registry. -nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries $flake2Dir#bar --refresh - -# Updating the flake should not change the lockfile. -nix flake lock $flake2Dir -[[ -z $(git -C $flake2Dir diff master) ]] - -# Now we should be able to build the flake in pure mode. -nix build -o $TEST_ROOT/result flake2#bar - -# Or without a registry. -nix build -o $TEST_ROOT/result --no-registries git+file://$flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries git+file://$flake2Dir#bar --refresh - -# Test whether indirect dependencies work. -nix build -o $TEST_ROOT/result $flake3Dir#xyzzy -git -C $flake3Dir add flake.lock - -# Add dependency to flake3. -rm $flake3Dir/flake.nix - -cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < \$out - [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]] - [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]] - ''; - }; - }; -} -EOF - -cp ./config.nix $flake3Dir - -git -C $flake3Dir add flake.nix config.nix -git -C $flake3Dir commit -m 'Add nonFlakeInputs' - -# Check whether `nix build` works with a lockfile which is missing a -# nonFlakeInputs. -nix build -o $TEST_ROOT/result $flake3Dir#sth --commit-lock-file - -nix build -o $TEST_ROOT/result flake3#fnord -[[ $(cat $TEST_ROOT/result) = FNORD ]] - -# Check whether flake input fetching is lazy: flake3#sth does not -# depend on flake2, so this shouldn't fail. -rm -rf $TEST_HOME/.cache -clearStore -mv $flake2Dir $flake2Dir.tmp -mv $nonFlakeDir $nonFlakeDir.tmp -nix build -o $TEST_ROOT/result flake3#sth -(! nix build -o $TEST_ROOT/result flake3#xyzzy) -(! nix build -o $TEST_ROOT/result flake3#fnord) -mv $flake2Dir.tmp $flake2Dir -mv $nonFlakeDir.tmp $nonFlakeDir -nix build -o $TEST_ROOT/result flake3#xyzzy flake3#fnord - -# Test doing multiple `lookupFlake`s -nix build -o $TEST_ROOT/result flake4#xyzzy - -# Test 'nix flake update' and --override-flake. -nix flake lock $flake3Dir -[[ -z $(git -C $flake3Dir diff master) ]] - -nix flake update $flake3Dir --override-flake flake2 nixpkgs -[[ ! -z $(git -C $flake3Dir diff master) ]] - -# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore -git -C $flake3Dir checkout -b removeXyzzy -rm $flake3Dir/flake.nix - -cat > $flake3Dir/flake.nix < \$out - ''; - }; - }; -} -EOF -nix flake lock $flake3Dir -git -C $flake3Dir add flake.nix flake.lock -git -C $flake3Dir commit -m 'Remove packages.xyzzy' -git -C $flake3Dir checkout master - -# Test whether fuzzy-matching works for registry entries. -(! nix build -o $TEST_ROOT/result flake4/removeXyzzy#xyzzy) -nix build -o $TEST_ROOT/result flake4/removeXyzzy#sth - -# Testing the nix CLI -nix registry add flake1 flake3 -[[ $(nix registry list | wc -l) == 8 ]] -nix registry pin flake1 -[[ $(nix registry list | wc -l) == 8 ]] -nix registry pin flake1 flake3 -[[ $(nix registry list | wc -l) == 8 ]] -nix registry remove flake1 -[[ $(nix registry list | wc -l) == 7 ]] - -# Test 'nix flake init'. -cat > $templatesDir/flake.nix < $templatesDir/trivial/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix <&1 && fail "nix flake check should have failed" || true) -echo "$checkRes" | grep -q "packages.system-1.default" -echo "$checkRes" | grep -q "packages.system-2.default" - -# Test 'follows' inputs. -cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake5Dir/flake.nix <&1 | grep 'NAR hash mismatch' - -# Test --override-input. -git -C $flake3Dir reset --hard -nix flake lock $flake3Dir --override-input flake2/flake1 file://$TEST_ROOT/flake.tar.gz -vvvvv -[[ $(jq .nodes.flake1_2.locked.url $flake3Dir/flake.lock) =~ flake.tar.gz ]] - -nix flake lock $flake3Dir --override-input flake2/flake1 flake1 -[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]] - -nix flake lock $flake3Dir --override-input flake2/flake1 flake1/master/$hash1 -[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash1 ]] - -# Test --update-input. -nix flake lock $flake3Dir -[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) = $hash1 ]] - -nix flake lock $flake3Dir --update-input flake2/flake1 -[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]] - -# Test 'nix flake metadata --json'. -nix flake metadata $flake3Dir --json | jq . - -# Test circular flake dependencies. -cat > $flakeA/flake.nix < $flakeB/flake.nix < $flakeFollowsA/flake.nix < $flakeFollowsB/flake.nix < $flakeFollowsC/flake.nix < $flakeFollowsD/flake.nix < $flakeFollowsE/flake.nix < $flakeFollowsA/flake.nix < $flakeFollowsA/flake.nix <&1 | grep 'points outside' - -# Test flake in store does not evaluate -rm -rf $badFlakeDir -mkdir $badFlakeDir -echo INVALID > $badFlakeDir/flake.nix -nix store delete $(nix store add-path $badFlakeDir) - -[[ $(nix path-info $(nix store add-path $flake1Dir)) =~ flake1 ]] -[[ $(nix path-info path:$(nix store add-path $flake1Dir)) =~ simple ]] diff -Nru nix-2.8.0/tests/fmt.sh nix-2.16.1+dfsg/tests/fmt.sh --- nix-2.8.0/tests/fmt.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/fmt.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,7 +1,5 @@ source common.sh -set -o pipefail - clearStore rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local @@ -18,7 +16,12 @@ with import ./config.nix; mkDerivation { name = "formatter"; - buildCommand = "mkdir -p \$out/bin; cp \${./fmt.simple.sh} \$out/bin/formatter"; + buildCommand = '' + mkdir -p \$out/bin + echo "#! ${shell}" > \$out/bin/formatter + cat \${./fmt.simple.sh} >> \$out/bin/formatter + chmod +x \$out/bin/formatter + ''; }; }; } diff -Nru nix-2.8.0/tests/function-trace.sh nix-2.16.1+dfsg/tests/function-trace.sh --- nix-2.8.0/tests/function-trace.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/function-trace.sh 2023-06-06 10:55:23.000000000 +0000 @@ -10,17 +10,15 @@ --trace-function-calls \ --expr "$expr" 2>&1 \ | grep "function-trace" \ - | sed -e 's/ [0-9]*$//' - ); + | sed -e 's/ [0-9]*$//' \ + || true + ) echo -n "Tracing expression '$expr'" - set +e msg=$(diff -swB \ <(echo "$expect") \ <(echo "$actual") - ); - result=$? - set -e + ) && result=0 || result=$? if [ $result -eq 0 ]; then echo " ok." else @@ -32,40 +30,38 @@ # failure inside a tryEval expect_trace 'builtins.tryEval (throw "example")' " -function-trace entered (string):1:1 at -function-trace entered (string):1:19 at -function-trace exited (string):1:19 at -function-trace exited (string):1:1 at +function-trace entered «string»:1:1 at +function-trace entered «string»:1:19 at +function-trace exited «string»:1:19 at +function-trace exited «string»:1:1 at " # Missing argument to a formal function expect_trace '({ x }: x) { }' " -function-trace entered (string):1:1 at -function-trace exited (string):1:1 at +function-trace entered «string»:1:1 at +function-trace exited «string»:1:1 at " # Too many arguments to a formal function expect_trace '({ x }: x) { x = "x"; y = "y"; }' " -function-trace entered (string):1:1 at -function-trace exited (string):1:1 at +function-trace entered «string»:1:1 at +function-trace exited «string»:1:1 at " # Not enough arguments to a lambda expect_trace '(x: y: x + y) 1' " -function-trace entered (string):1:1 at -function-trace exited (string):1:1 at +function-trace entered «string»:1:1 at +function-trace exited «string»:1:1 at " # Too many arguments to a lambda expect_trace '(x: x) 1 2' " -function-trace entered (string):1:1 at -function-trace exited (string):1:1 at +function-trace entered «string»:1:1 at +function-trace exited «string»:1:1 at " # Not a function expect_trace '1 2' " -function-trace entered (string):1:1 at -function-trace exited (string):1:1 at +function-trace entered «string»:1:1 at +function-trace exited «string»:1:1 at " - -set -e diff -Nru nix-2.8.0/tests/gc-runtime.sh nix-2.16.1+dfsg/tests/gc-runtime.sh --- nix-2.8.0/tests/gc-runtime.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/gc-runtime.sh 2023-06-06 10:55:23.000000000 +0000 @@ -4,7 +4,7 @@ *linux*) ;; *) - exit 99; + skipTest "Not running Linux"; esac set -m # enable job control, needed for kill diff -Nru nix-2.8.0/tests/gc.sh nix-2.16.1+dfsg/tests/gc.sh --- nix-2.8.0/tests/gc.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/gc.sh 2023-06-06 10:55:23.000000000 +0000 @@ -50,3 +50,31 @@ # Check that the store is empty. rmdir $NIX_STORE_DIR/.links rmdir $NIX_STORE_DIR + +## Test `nix-collect-garbage -d` +testCollectGarbageD () { + clearProfiles + # Run two `nix-env` commands, should create two generations of + # the profile + nix-env -f ./user-envs.nix -i foo-1.0 + nix-env -f ./user-envs.nix -i foo-2.0pre1 + [[ $(nix-env --list-generations | wc -l) -eq 2 ]] + + # Clear the profile history. There should be only one generation + # left + nix-collect-garbage -d + [[ $(nix-env --list-generations | wc -l) -eq 1 ]] +} +# `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're +# using them +if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + testCollectGarbageD + + # Run the same test, but forcing the profiles at their legacy location under + # /nix/var/nix. + # + # Regression test for #8294 + rm ~/.nix-profile + ln -s $NIX_STATE_DIR/profiles/per-user/me ~/.nix-profile + testCollectGarbageD +fi diff -Nru nix-2.8.0/tests/github-flakes.nix nix-2.16.1+dfsg/tests/github-flakes.nix --- nix-2.8.0/tests/github-flakes.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/github-flakes.nix 1970-01-01 00:00:00.000000000 +0000 @@ -1,150 +0,0 @@ -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - -let - - # Generate a fake root CA and a fake github.com certificate. - cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } - '' - mkdir -p $out - - openssl genrsa -out ca.key 2048 - openssl req -new -x509 -days 36500 -key ca.key \ - -subj "/C=NL/ST=Denial/L=Springfield/O=Dis/CN=Root CA" -out $out/ca.crt - - openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \ - -subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=github.com" -out server.csr - openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:github.com,DNS:raw.githubusercontent.com") \ - -days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt - ''; - - registry = pkgs.writeTextFile { - name = "registry"; - text = '' - { - "flakes": [ - { - "from": { - "type": "indirect", - "id": "nixpkgs" - }, - "to": { - "type": "github", - "owner": "NixOS", - "repo": "nixpkgs" - } - } - ], - "version": 2 - } - ''; - destination = "/flake-registry.json"; - }; - - api = pkgs.runCommand "nixpkgs-flake" {} - '' - mkdir -p $out/tarball - - dir=NixOS-nixpkgs-${nixpkgs.shortRev} - cp -prd ${nixpkgs} $dir - # Set the correct timestamp in the tarball. - find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- - tar cfz $out/tarball/${nixpkgs.rev} $dir --hard-dereference - - mkdir -p $out/commits - echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD - ''; - -in - -makeTest ( - -{ - name = "github-flakes"; - - nodes = - { # Impersonate github.com and api.github.com. - github = - { config, pkgs, ... }: - { networking.firewall.allowedTCPPorts = [ 80 443 ]; - - services.httpd.enable = true; - services.httpd.adminAddr = "foo@example.org"; - services.httpd.extraConfig = '' - ErrorLog syslog:local6 - ''; - services.httpd.virtualHosts."github.com" = - { forceSSL = true; - sslServerKey = "${cert}/server.key"; - sslServerCert = "${cert}/server.crt"; - servedDirs = - [ { urlPath = "/NixOS/flake-registry/raw/master"; - dir = registry; - } - ]; - }; - services.httpd.virtualHosts."api.github.com" = - { forceSSL = true; - sslServerKey = "${cert}/server.key"; - sslServerCert = "${cert}/server.crt"; - servedDirs = - [ { urlPath = "/repos/NixOS/nixpkgs"; - dir = api; - } - ]; - }; - }; - - client = - { config, lib, pkgs, nodes, ... }: - { virtualisation.writableStore = true; - virtualisation.diskSize = 2048; - virtualisation.pathsInNixDB = [ pkgs.hello pkgs.fuse ]; - virtualisation.memorySize = 4096; - nix.binaryCaches = lib.mkForce [ ]; - nix.extraOptions = "experimental-features = nix-command flakes"; - environment.systemPackages = [ pkgs.jq ]; - networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} = - [ "github.com" "api.github.com" "raw.githubusercontent.com" ]; - security.pki.certificateFiles = [ "${cert}/ca.crt" ]; - }; - }; - - testScript = { nodes }: '' - # fmt: off - import json - import time - - start_all() - - github.wait_for_unit("httpd.service") - - client.succeed("curl -v https://github.com/ >&2") - client.succeed("nix registry list | grep nixpkgs") - - rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision") - assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" - - client.succeed("nix registry pin nixpkgs") - - client.succeed("nix flake info nixpkgs --tarball-ttl 0 >&2") - - # Shut down the web server. The flake should be cached on the client. - github.succeed("systemctl stop httpd.service") - - info = json.loads(client.succeed("nix flake info nixpkgs --json")) - date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified'])) - assert date == "${nixpkgs.lastModifiedDate}", "time mismatch" - - client.succeed("nix build nixpkgs#hello") - - # The build shouldn't fail even with --tarball-ttl 0 (the server - # being down should not be a fatal error). - client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") - ''; - -}) diff -Nru nix-2.8.0/tests/hash.sh nix-2.16.1+dfsg/tests/hash.sh --- nix-2.8.0/tests/hash.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/hash.sh 2023-06-06 10:55:23.000000000 +0000 @@ -2,13 +2,19 @@ try () { printf "%s" "$2" > $TEST_ROOT/vector - hash=$(nix hash file --base16 $EXTRA --type "$1" $TEST_ROOT/vector) - if test "$hash" != "$3"; then - echo "hash $1, expected $3, got $hash" + hash="$(nix-hash --flat ${FORMAT_FLAG-} --type "$1" "$TEST_ROOT/vector")" + if ! (( "${NO_TEST_CLASSIC-}" )) && test "$hash" != "$3"; then + echo "try nix-hash: hash $1, expected $3, got $hash" + exit 1 + fi + hash="$(nix hash file ${FORMAT_FLAG-} --type "$1" "$TEST_ROOT/vector")" + if ! (( "${NO_TEST_NIX_COMMAND-}" )) && test "$hash" != "$3"; then + echo "try nix hash: hash $1, expected $3, got $hash" exit 1 fi } +FORMAT_FLAG=--base16 try md5 "" "d41d8cd98f00b204e9800998ecf8427e" try md5 "a" "0cc175b9c0f1b6a831c399e269772661" try md5 "abc" "900150983cd24fb0d6963f7d28e17f72" @@ -28,16 +34,24 @@ try sha512 "" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" +unset FORMAT_FLAG -EXTRA=--base32 +FORMAT_FLAG=--base32 try sha256 "abc" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" -EXTRA= +unset FORMAT_FLAG -EXTRA=--sri +FORMAT_FLAG=--sri try sha512 "" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha256-JI1qYdIGOLjlwCaTDD5gOaM85Flk/yFn9uzt1BnbBsE=" +unset FORMAT_FLAG + +# nix-hash [--flat] defaults to the Base16 format +NO_TEST_NIX_COMMAND=1 try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" + +# nix hash [file|path] defaults to the SRI format +NO_TEST_CLASSIC=1 try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" try2 () { hash=$(nix-hash --type "$1" $TEST_ROOT/hash-path) @@ -69,12 +83,18 @@ # Conversion. try3() { + h64=$(nix-hash --type "$1" --to-base64 "$2") + [ "$h64" = "$4" ] h64=$(nix hash to-base64 --type "$1" "$2") [ "$h64" = "$4" ] + sri=$(nix-hash --type "$1" --to-sri "$2") + [ "$sri" = "$1-$4" ] sri=$(nix hash to-sri --type "$1" "$2") [ "$sri" = "$1-$4" ] h32=$(nix-hash --type "$1" --to-base32 "$2") [ "$h32" = "$3" ] + h32=$(nix hash to-base32 --type "$1" "$2") + [ "$h32" = "$3" ] h16=$(nix-hash --type "$1" --to-base16 "$h32") [ "$h16" = "$2" ] h16=$(nix hash to-base16 --type "$1" "$h64") diff -Nru nix-2.8.0/tests/impure-derivations.sh nix-2.16.1+dfsg/tests/impure-derivations.sh --- nix-2.8.0/tests/impure-derivations.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/impure-derivations.sh 2023-06-06 10:55:23.000000000 +0000 @@ -2,16 +2,24 @@ requireDaemonNewerThan "2.8pre20220311" -enableFeatures "ca-derivations ca-references impure-derivations" +enableFeatures "ca-derivations impure-derivations" restartDaemon -set -o pipefail - clearStore # Basic test of impure derivations: building one a second time should not use the previous result. printf 0 > $TEST_ROOT/counter +# `nix derivation add` with impure derivations work +drvPath=$(nix-instantiate ./impure-derivations.nix -A impure) +nix derivation show $drvPath | jq .[] > $TEST_HOME/impure-drv.json +drvPath2=$(nix derivation add < $TEST_HOME/impure-drv.json) +[[ "$drvPath" = "$drvPath2" ]] + +# But only with the experimental feature! +expectStderr 1 nix derivation add < $TEST_HOME/impure-drv.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'impure-derivations' is disabled" + +nix build --dry-run --json --file ./impure-derivations.nix impure.all json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all) path1=$(echo $json | jq -r .[].outputs.out) path1_stuff=$(echo $json | jq -r .[].outputs.stuff) @@ -38,8 +46,8 @@ (! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation' drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .) -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] # Fixed-output derivations *can* depend on impure derivations. path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) diff -Nru nix-2.8.0/tests/init.sh nix-2.16.1+dfsg/tests/init.sh --- nix-2.8.0/tests/init.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/init.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,8 +1,11 @@ -source common.sh +# Don't start the daemon +source common/vars-and-functions.sh test -n "$TEST_ROOT" if test -d "$TEST_ROOT"; then chmod -R u+w "$TEST_ROOT" + # We would delete any daemon socket, so let's stop the daemon first. + killDaemon rm -rf "$TEST_ROOT" fi mkdir "$TEST_ROOT" diff -Nru nix-2.8.0/tests/install-darwin.sh nix-2.16.1+dfsg/tests/install-darwin.sh --- nix-2.8.0/tests/install-darwin.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/install-darwin.sh 2023-06-06 10:55:23.000000000 +0000 @@ -4,7 +4,7 @@ cleanup() { PLIST="/Library/LaunchDaemons/org.nixos.nix-daemon.plist" - if sudo launchctl list | grep -q nix-daemon; then + if sudo launchctl list | grepQuiet nix-daemon; then sudo launchctl unload "$PLIST" fi diff -Nru nix-2.8.0/tests/installer/default.nix nix-2.16.1+dfsg/tests/installer/default.nix --- nix-2.8.0/tests/installer/default.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/installer/default.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,244 @@ +{ binaryTarballs +, nixpkgsFor +}: + +let + + installScripts = { + install-default = { + script = '' + tar -xf ./nix.tar.xz + mv ./nix-* nix + ./nix/install --no-channel-add + ''; + }; + + install-force-no-daemon = { + script = '' + tar -xf ./nix.tar.xz + mv ./nix-* nix + ./nix/install --no-daemon --no-channel-add + ''; + }; + + install-force-daemon = { + script = '' + tar -xf ./nix.tar.xz + mv ./nix-* nix + ./nix/install --daemon --no-channel-add + ''; + }; + }; + + mockChannel = pkgs: + pkgs.runCommandNoCC "mock-channel" {} '' + mkdir nixexprs + mkdir -p $out/channel + echo -n 'someContent' > nixexprs/someFile + tar cvf - nixexprs | bzip2 > $out/channel/nixexprs.tar.bz2 + ''; + + disableSELinux = "sudo setenforce 0"; + + images = { + + /* + "ubuntu-14-04" = { + image = import { + url = "https://app.vagrantup.com/ubuntu/boxes/trusty64/versions/20190514.0.0/providers/virtualbox.box"; + hash = "sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="; + }; + rootDisk = "box-disk1.vmdk"; + system = "x86_64-linux"; + }; + */ + + "ubuntu-16-04" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/ubuntu1604/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-lO4oYQR2tCh5auxAYe6bPOgEqOgv3Y3GC1QM1tEEEU8="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + }; + + "ubuntu-22-04" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/ubuntu2204/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-HNll0Qikw/xGIcogni5lz01vUv+R3o8xowP2EtqjuUQ="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + }; + + "fedora-36" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/fedora36/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-rxPgnDnFkTDwvdqn2CV3ZUo3re9AdPtSZ9SvOHNvaks="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + postBoot = disableSELinux; + }; + + # Currently fails with 'error while loading shared libraries: + # libsodium.so.23: cannot stat shared object: Invalid argument'. + /* + "rhel-6" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/rhel6/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-QwzbvRoRRGqUCQptM7X/InRWFSP2sqwRt2HaaO6zBGM="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + }; + */ + + "rhel-7" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/rhel7/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + }; + + "rhel-8" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/rhel8/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-zFOPjSputy1dPgrQRixBXmlyN88cAKjJ21VvjSWUCUY="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + postBoot = disableSELinux; + }; + + "rhel-9" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/rhel9/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-vL/FbB3kK1rcSaR627nWmScYGKGk4seSmAdq6N5diMg="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + postBoot = disableSELinux; + extraQemuOpts = "-cpu Westmere-v2"; + }; + + }; + + makeTest = imageName: testName: + let image = images.${imageName}; in + with nixpkgsFor.${image.system}.native; + runCommand + "installer-test-${imageName}-${testName}" + { buildInputs = [ qemu_kvm openssh ]; + image = image.image; + postBoot = image.postBoot or ""; + installScript = installScripts.${testName}.script; + binaryTarball = binaryTarballs.${system}; + } + '' + shopt -s nullglob + + echo "Unpacking Vagrant box $image..." + tar xvf $image + + image_type=$(qemu-img info ${image.rootDisk} | sed 's/file format: \(.*\)/\1/; t; d') + + qemu-img create -b ./${image.rootDisk} -F "$image_type" -f qcow2 ./disk.qcow2 + + extra_qemu_opts="${image.extraQemuOpts or ""}" + + # Add the config disk, required by the Ubuntu images. + config_drive=$(echo *configdrive.vmdk || true) + if [[ -n $config_drive ]]; then + extra_qemu_opts+=" -drive id=disk2,file=$config_drive,if=virtio" + fi + + echo "Starting qemu..." + qemu-kvm -m 4096 -nographic \ + -drive id=disk1,file=./disk.qcow2,if=virtio \ + -netdev user,id=net0,restrict=yes,hostfwd=tcp::20022-:22 -device virtio-net-pci,netdev=net0 \ + $extra_qemu_opts & + qemu_pid=$! + trap "kill $qemu_pid" EXIT + + if ! [ -e ./vagrant_insecure_key ]; then + cp ${./vagrant_insecure_key} vagrant_insecure_key + fi + + chmod 0400 ./vagrant_insecure_key + + ssh_opts="-o StrictHostKeyChecking=no -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa -i ./vagrant_insecure_key" + ssh="ssh -p 20022 -q $ssh_opts vagrant@localhost" + + echo "Waiting for SSH..." + for ((i = 0; i < 120; i++)); do + echo "[ssh] Trying to connect..." + if $ssh -- true; then + echo "[ssh] Connected!" + break + fi + if ! kill -0 $qemu_pid; then + echo "qemu died unexpectedly" + exit 1 + fi + sleep 1 + done + + if [[ -n $postBoot ]]; then + echo "Running post-boot commands..." + $ssh "set -ex; $postBoot" + fi + + echo "Copying installer..." + scp -P 20022 $ssh_opts $binaryTarball/nix-*.tar.xz vagrant@localhost:nix.tar.xz + + echo "Running installer..." + $ssh "set -eux; $installScript" + + echo "Copying the mock channel" + # `scp -r` doesn't seem to work properly on some rhel instances, so let's + # use a plain tarpipe instead + tar -C ${mockChannel pkgs} -c channel | ssh -p 20022 $ssh_opts vagrant@localhost tar x -f- + + echo "Testing Nix installation..." + $ssh < \$out"]; }') + [[ \$(cat \$out) = foobar ]] + + if pgrep nix-daemon; then + MAYBESUDO="sudo" + else + MAYBESUDO="" + fi + + + $MAYBESUDO \$(which nix-channel) --add file://\$HOME/channel myChannel + $MAYBESUDO \$(which nix-channel) --update + [[ \$(nix-instantiate --eval --expr 'builtins.readFile ') = '"someContent"' ]] + EOF + + echo "Done!" + touch $out + ''; + +in + +builtins.mapAttrs (imageName: image: + { ${image.system} = builtins.mapAttrs (testName: test: + makeTest imageName testName + ) installScripts; + } +) images diff -Nru nix-2.8.0/tests/installer/vagrant_insecure_key nix-2.16.1+dfsg/tests/installer/vagrant_insecure_key --- nix-2.8.0/tests/installer/vagrant_insecure_key 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/installer/vagrant_insecure_key 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI +w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP +kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 +hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO +Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW +yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd +ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 +Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf +TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK +iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A +sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf +4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP +cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk +EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN +CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX +3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG +YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj +3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ +dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz +6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC +P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF +llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ +kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH ++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ +NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= +-----END RSA PRIVATE KEY----- diff -Nru nix-2.8.0/tests/lang/eval-fail-foldlStrict-strict-op-application.nix nix-2.16.1+dfsg/tests/lang/eval-fail-foldlStrict-strict-op-application.nix --- nix-2.8.0/tests/lang/eval-fail-foldlStrict-strict-op-application.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-fail-foldlStrict-strict-op-application.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,5 @@ +# Tests that the result of applying op is forced even if the value is never used +builtins.foldl' + (_: f: f null) + null + [ (_: throw "Not the final value, but is still forced!") (_: 23) ] diff -Nru nix-2.8.0/tests/lang/eval-okay-closure.exp nix-2.16.1+dfsg/tests/lang/eval-okay-closure.exp --- nix-2.8.0/tests/lang/eval-okay-closure.exp 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-closure.exp 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1 @@ +[ { foo = true; key = -13; } { foo = true; key = -12; } { foo = true; key = -11; } { foo = true; key = -9; } { foo = true; key = -8; } { foo = true; key = -7; } { foo = true; key = -5; } { foo = true; key = -4; } { foo = true; key = -3; } { key = -1; } { foo = true; key = 0; } { foo = true; key = 1; } { foo = true; key = 2; } { foo = true; key = 4; } { foo = true; key = 5; } { foo = true; key = 6; } { key = 8; } { foo = true; key = 9; } { foo = true; key = 10; } { foo = true; key = 13; } { foo = true; key = 14; } { foo = true; key = 15; } { key = 17; } { foo = true; key = 18; } { foo = true; key = 19; } { foo = true; key = 22; } { foo = true; key = 23; } { key = 26; } { foo = true; key = 27; } { foo = true; key = 28; } { foo = true; key = 31; } { foo = true; key = 32; } { key = 35; } { foo = true; key = 36; } { foo = true; key = 40; } { foo = true; key = 41; } { key = 44; } { foo = true; key = 45; } { foo = true; key = 49; } { key = 53; } { foo = true; key = 54; } { foo = true; key = 58; } { key = 62; } { foo = true; key = 67; } { key = 71; } { key = 80; } ] diff -Nru nix-2.8.0/tests/lang/eval-okay-context-introspection.exp nix-2.16.1+dfsg/tests/lang/eval-okay-context-introspection.exp --- nix-2.8.0/tests/lang/eval-okay-context-introspection.exp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-context-introspection.exp 2023-06-06 10:55:23.000000000 +0000 @@ -1 +1 @@ -true +[ true true true true true true ] diff -Nru nix-2.8.0/tests/lang/eval-okay-context-introspection.nix nix-2.16.1+dfsg/tests/lang/eval-okay-context-introspection.nix --- nix-2.8.0/tests/lang/eval-okay-context-introspection.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-context-introspection.nix 2023-06-06 10:55:23.000000000 +0000 @@ -18,7 +18,24 @@ }; }; - legit-context = builtins.getContext "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}"; + combo-path = "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}"; + legit-context = builtins.getContext combo-path; - constructed-context = builtins.getContext (builtins.appendContext "" desired-context); -in legit-context == constructed-context + reconstructed-path = builtins.appendContext + (builtins.unsafeDiscardStringContext combo-path) + desired-context; + + # Eta rule for strings with context. + etaRule = str: + str == builtins.appendContext + (builtins.unsafeDiscardStringContext str) + (builtins.getContext str); + +in [ + (legit-context == desired-context) + (reconstructed-path == combo-path) + (etaRule "foo") + (etaRule drv.drvPath) + (etaRule drv.foo.outPath) + (etaRule (builtins.unsafeDiscardOutputDependency drv.drvPath)) +] diff -Nru nix-2.8.0/tests/lang/eval-okay-eq.exp nix-2.16.1+dfsg/tests/lang/eval-okay-eq.exp --- nix-2.8.0/tests/lang/eval-okay-eq.exp 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-eq.exp 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1 @@ +true diff -Nru nix-2.8.0/tests/lang/eval-okay-eq.exp.disabled nix-2.16.1+dfsg/tests/lang/eval-okay-eq.exp.disabled --- nix-2.8.0/tests/lang/eval-okay-eq.exp.disabled 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-eq.exp.disabled 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -Bool(True) diff -Nru nix-2.8.0/tests/lang/eval-okay-foldlStrict-lazy-elements.exp nix-2.16.1+dfsg/tests/lang/eval-okay-foldlStrict-lazy-elements.exp --- nix-2.8.0/tests/lang/eval-okay-foldlStrict-lazy-elements.exp 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-foldlStrict-lazy-elements.exp 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1 @@ +42 diff -Nru nix-2.8.0/tests/lang/eval-okay-foldlStrict-lazy-elements.nix nix-2.16.1+dfsg/tests/lang/eval-okay-foldlStrict-lazy-elements.nix --- nix-2.8.0/tests/lang/eval-okay-foldlStrict-lazy-elements.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-foldlStrict-lazy-elements.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,9 @@ +# Tests that the rhs argument of op is not forced unconditionally +let + lst = builtins.foldl' + (acc: x: acc ++ [ x ]) + [ ] + [ 42 (throw "this shouldn't be evaluated") ]; +in + +builtins.head lst diff -Nru nix-2.8.0/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp nix-2.16.1+dfsg/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp --- nix-2.8.0/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1 @@ +42 diff -Nru nix-2.8.0/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix nix-2.16.1+dfsg/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix --- nix-2.8.0/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,6 @@ +# Checks that the nul value for the accumulator is not forced unconditionally. +# Some languages provide a foldl' that is strict in this argument, but Nix does not. +builtins.foldl' + (_: x: x) + (throw "This is never forced") + [ "but the results of applying op are" 42 ] diff -Nru nix-2.8.0/tests/lang/eval-okay-functionargs.exp nix-2.16.1+dfsg/tests/lang/eval-okay-functionargs.exp --- nix-2.8.0/tests/lang/eval-okay-functionargs.exp 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-functionargs.exp 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1 @@ +[ "stdenv" "fetchurl" "aterm-stdenv" "aterm-stdenv2" "libX11" "libXv" "mplayer-stdenv2.libXv-libX11" "mplayer-stdenv2.libXv-libX11_2" "nix-stdenv-aterm-stdenv" "nix-stdenv2-aterm2-stdenv2" ] diff -Nru nix-2.8.0/tests/lang/eval-okay-ind-string.nix nix-2.16.1+dfsg/tests/lang/eval-okay-ind-string.nix --- nix-2.8.0/tests/lang/eval-okay-ind-string.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-ind-string.nix 2023-06-06 10:55:23.000000000 +0000 @@ -110,7 +110,7 @@ And finally to interpret \n etc. as in a string: ''\n, ''\r, ''\t. ''; - # Regression test: antiquotation in '${x}' should work, but didn't. + # Regression test: string interpolation in '${x}' should work, but didn't. s15 = let x = "bla"; in '' foo '${x}' diff -Nru nix-2.8.0/tests/lang/eval-okay-intersectAttrs.exp nix-2.16.1+dfsg/tests/lang/eval-okay-intersectAttrs.exp --- nix-2.8.0/tests/lang/eval-okay-intersectAttrs.exp 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-intersectAttrs.exp 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1 @@ +[ { } { a = 1; } { a = 1; } { a = "a"; } { m = 1; } { m = "m"; } { n = 1; } { n = "n"; } { n = 1; p = 2; } { n = "n"; p = "p"; } { n = 1; p = 2; } { n = "n"; p = "p"; } { a = "a"; b = "b"; c = "c"; d = "d"; e = "e"; f = "f"; g = "g"; h = "h"; i = "i"; j = "j"; k = "k"; l = "l"; m = "m"; n = "n"; o = "o"; p = "p"; q = "q"; r = "r"; s = "s"; t = "t"; u = "u"; v = "v"; w = "w"; x = "x"; y = "y"; z = "z"; } true ] diff -Nru nix-2.8.0/tests/lang/eval-okay-intersectAttrs.nix nix-2.16.1+dfsg/tests/lang/eval-okay-intersectAttrs.nix --- nix-2.8.0/tests/lang/eval-okay-intersectAttrs.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-intersectAttrs.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,50 @@ +let + alphabet = + { a = "a"; + b = "b"; + c = "c"; + d = "d"; + e = "e"; + f = "f"; + g = "g"; + h = "h"; + i = "i"; + j = "j"; + k = "k"; + l = "l"; + m = "m"; + n = "n"; + o = "o"; + p = "p"; + q = "q"; + r = "r"; + s = "s"; + t = "t"; + u = "u"; + v = "v"; + w = "w"; + x = "x"; + y = "y"; + z = "z"; + }; + foo = { + inherit (alphabet) f o b a r z q u x; + aa = throw "aa"; + }; + alphabetFail = builtins.mapAttrs throw alphabet; +in +[ (builtins.intersectAttrs { a = abort "l1"; } { b = abort "r1"; }) + (builtins.intersectAttrs { a = abort "l2"; } { a = 1; }) + (builtins.intersectAttrs alphabetFail { a = 1; }) + (builtins.intersectAttrs { a = abort "laa"; } alphabet) + (builtins.intersectAttrs alphabetFail { m = 1; }) + (builtins.intersectAttrs { m = abort "lam"; } alphabet) + (builtins.intersectAttrs alphabetFail { n = 1; }) + (builtins.intersectAttrs { n = abort "lan"; } alphabet) + (builtins.intersectAttrs alphabetFail { n = 1; p = 2; }) + (builtins.intersectAttrs { n = abort "lan2"; p = abort "lap"; } alphabet) + (builtins.intersectAttrs alphabetFail { n = 1; p = 2; }) + (builtins.intersectAttrs { n = abort "lan2"; p = abort "lap"; } alphabet) + (builtins.intersectAttrs alphabetFail alphabet) + (builtins.intersectAttrs alphabet foo == builtins.intersectAttrs foo alphabet) +] diff -Nru nix-2.8.0/tests/lang/eval-okay-path-antiquotation.exp nix-2.16.1+dfsg/tests/lang/eval-okay-path-antiquotation.exp --- nix-2.8.0/tests/lang/eval-okay-path-antiquotation.exp 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-path-antiquotation.exp 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1 @@ +{ absolute = /foo; expr = /pwd/lang/foo/bar; home = /fake-home/foo; notfirst = /pwd/lang/bar/foo; simple = /pwd/lang/foo; slashes = /foo/bar; surrounded = /pwd/lang/a-foo-b; } diff -Nru nix-2.8.0/tests/lang/eval-okay-path.exp nix-2.16.1+dfsg/tests/lang/eval-okay-path.exp --- nix-2.8.0/tests/lang/eval-okay-path.exp 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-path.exp 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1 @@ +"/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" diff -Nru nix-2.8.0/tests/lang/eval-okay-readDir.exp nix-2.16.1+dfsg/tests/lang/eval-okay-readDir.exp --- nix-2.8.0/tests/lang/eval-okay-readDir.exp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-readDir.exp 2023-06-06 10:55:23.000000000 +0000 @@ -1 +1 @@ -{ bar = "regular"; foo = "directory"; } +{ bar = "regular"; foo = "directory"; ldir = "symlink"; linked = "symlink"; } diff -Nru nix-2.8.0/tests/lang/eval-okay-readFileType.exp nix-2.16.1+dfsg/tests/lang/eval-okay-readFileType.exp --- nix-2.8.0/tests/lang/eval-okay-readFileType.exp 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-readFileType.exp 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1 @@ +{ bar = "regular"; foo = "directory"; ldir = "symlink"; linked = "symlink"; } diff -Nru nix-2.8.0/tests/lang/eval-okay-readFileType.nix nix-2.16.1+dfsg/tests/lang/eval-okay-readFileType.nix --- nix-2.8.0/tests/lang/eval-okay-readFileType.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-readFileType.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,6 @@ +{ + bar = builtins.readFileType ./readDir/bar; + foo = builtins.readFileType ./readDir/foo; + linked = builtins.readFileType ./readDir/linked; + ldir = builtins.readFileType ./readDir/ldir; +} diff -Nru nix-2.8.0/tests/lang/eval-okay-replacestrings.exp nix-2.16.1+dfsg/tests/lang/eval-okay-replacestrings.exp --- nix-2.8.0/tests/lang/eval-okay-replacestrings.exp 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-replacestrings.exp 2023-06-06 10:55:23.000000000 +0000 @@ -1 +1 @@ -[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" ] +[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" "fubar" ] diff -Nru nix-2.8.0/tests/lang/eval-okay-replacestrings.nix nix-2.16.1+dfsg/tests/lang/eval-okay-replacestrings.nix --- nix-2.8.0/tests/lang/eval-okay-replacestrings.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-replacestrings.nix 2023-06-06 10:55:23.000000000 +0000 @@ -8,4 +8,5 @@ (replaceStrings [""] ["X"] "abc") (replaceStrings [""] ["X"] "") (replaceStrings ["-"] ["_"] "a-b") + (replaceStrings ["oo" "XX"] ["u" (throw "unreachable")] "foobar") ] diff -Nru nix-2.8.0/tests/lang/eval-okay-versions.nix nix-2.16.1+dfsg/tests/lang/eval-okay-versions.nix --- nix-2.8.0/tests/lang/eval-okay-versions.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/eval-okay-versions.nix 2023-06-06 10:55:23.000000000 +0000 @@ -4,6 +4,7 @@ name2 = "hello"; name3 = "915resolution-0.5.2"; name4 = "xf86-video-i810-1.7.4"; + name5 = "name-that-ends-with-dash--1.0"; eq = 0; lt = builtins.sub 0 1; @@ -23,6 +24,8 @@ ((builtins.parseDrvName name3).version == "0.5.2") ((builtins.parseDrvName name4).name == "xf86-video-i810") ((builtins.parseDrvName name4).version == "1.7.4") + ((builtins.parseDrvName name5).name == "name-that-ends-with-dash") + ((builtins.parseDrvName name5).version == "-1.0") (versionTest "1.0" "2.3" lt) (versionTest "2.1" "2.3" lt) (versionTest "2.3" "2.3" eq) diff -Nru nix-2.8.0/tests/lang/parse-fail-eof-in-string.nix nix-2.16.1+dfsg/tests/lang/parse-fail-eof-in-string.nix --- nix-2.8.0/tests/lang/parse-fail-eof-in-string.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang/parse-fail-eof-in-string.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,3 @@ +# https://github.com/NixOS/nix/issues/6562 +# Note that this file must not end with a newline. +a 1"$ \ No newline at end of file diff -Nru nix-2.8.0/tests/lang.sh nix-2.16.1+dfsg/tests/lang.sh --- nix-2.8.0/tests/lang.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/lang.sh 2023-06-06 10:55:23.000000000 +0000 @@ -2,10 +2,21 @@ export TEST_VAR=foo # for eval-okay-getenv.nix export NIX_REMOTE=dummy:// +export NIX_STORE_DIR=/nix/store -nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello -(! nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grep -q Hello) -nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' 2>&1 | grep -q Hello +nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grepQuiet Hello +nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>/dev/null | grepQuiet 123 +nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 +nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuiet Hello +nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuietInverse Hello +nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello +expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' | grepQuiet Hello + +nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \ + 2>&1 | grepQuiet -E 'trace: { x = «potential infinite recursion»; }' + +nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x true; }; in x.tracing'\ + 2>&1 | grepQuiet -F 'trace: { repeating = «repeated»; tracing = «potential infinite recursion»; }' set +x @@ -14,7 +25,7 @@ for i in lang/parse-fail-*.nix; do echo "parsing $i (should fail)"; i=$(basename $i .nix) - if nix-instantiate --parse - < lang/$i.nix; then + if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then echo "FAIL: $i shouldn't parse" fail=1 fi @@ -23,7 +34,7 @@ for i in lang/parse-okay-*.nix; do echo "parsing $i (should succeed)"; i=$(basename $i .nix) - if ! nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then + if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then echo "FAIL: $i should parse" fail=1 fi @@ -32,7 +43,7 @@ for i in lang/eval-fail-*.nix; do echo "evaluating $i (should fail)"; i=$(basename $i .nix) - if nix-instantiate --eval lang/$i.nix; then + if ! expect 1 nix-instantiate --eval lang/$i.nix; then echo "FAIL: $i shouldn't evaluate" fail=1 fi @@ -47,17 +58,17 @@ if test -e lang/$i.flags; then flags=$(cat lang/$i.flags) fi - if ! NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then + if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 HOME=/fake-home nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then echo "FAIL: $i should evaluate" fail=1 - elif ! diff lang/$i.out lang/$i.exp; then + elif ! diff <(< lang/$i.out sed -e "s|$(pwd)|/pwd|g") lang/$i.exp; then echo "FAIL: evaluation result of $i not as expected" fail=1 fi fi if test -e lang/$i.exp.xml; then - if ! nix-instantiate --eval --xml --no-location --strict \ + if ! expect 0 nix-instantiate --eval --xml --no-location --strict \ lang/$i.nix > lang/$i.out.xml; then echo "FAIL: $i should evaluate" fail=1 diff -Nru nix-2.8.0/tests/legacy-ssh-store.sh nix-2.16.1+dfsg/tests/legacy-ssh-store.sh --- nix-2.8.0/tests/legacy-ssh-store.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/legacy-ssh-store.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,4 @@ +source common.sh + +# Check that store ping trusted doesn't yet work with ssh:// +nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e 'has("trusted") | not' diff -Nru nix-2.8.0/tests/linux-sandbox-cert-test.nix nix-2.16.1+dfsg/tests/linux-sandbox-cert-test.nix --- nix-2.8.0/tests/linux-sandbox-cert-test.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/linux-sandbox-cert-test.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,29 @@ +{ fixed-output }: + +with import ./config.nix; + +mkDerivation ({ + name = "ssl-export"; + buildCommand = '' + # Add some indirection, otherwise grepping into the debug output finds the string. + report () { echo CERT_$1_IN_SANDBOX; } + + if [ -f /etc/ssl/certs/ca-certificates.crt ]; then + content=$( $TEST_ROOT/log) -if grep -q 'error: renaming' $TEST_ROOT/log; then false; fi -grep -q 'may not be deterministic' $TEST_ROOT/log +if grepQuiet 'error: renaming' $TEST_ROOT/log; then false; fi +grepQuiet 'may not be deterministic' $TEST_ROOT/log + +# Test that sandboxed builds cannot write to /etc easily +(! nix-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' --no-out-link --sandbox-paths /nix/store) + + +## Test mounting of SSL certificates into the sandbox +testCert () { + (! nix-build linux-sandbox-cert-test.nix --argstr fixed-output "$2" --no-out-link --sandbox-paths /nix/store --option ssl-cert-file "$3" 2> $TEST_ROOT/log) + cat $TEST_ROOT/log + grepQuiet "CERT_${1}_IN_SANDBOX" $TEST_ROOT/log +} + +nocert=$TEST_ROOT/no-cert-file.pem +cert=$TEST_ROOT/some-cert-file.pem +echo -n "CERT_CONTENT" > $cert + +# No cert in sandbox when not a fixed-output derivation +testCert missing normal "$cert" + +# No cert in sandbox when ssl-cert-file is empty +testCert missing fixed-output "" + +# No cert in sandbox when ssl-cert-file is a nonexistent file +testCert missing fixed-output "$nocert" + +# Cert in sandbox when ssl-cert-file is set to an existing file +testCert present fixed-output "$cert" diff -Nru nix-2.8.0/tests/local.mk nix-2.16.1+dfsg/tests/local.mk --- nix-2.8.0/tests/local.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -1,16 +1,33 @@ nix_tests = \ - flakes.sh \ - flakes-run.sh \ + test-infra.sh \ + init.sh \ + flakes/flakes.sh \ + flakes/run.sh \ + flakes/mercurial.sh \ + flakes/circular.sh \ + flakes/init.sh \ + flakes/inputs.sh \ + flakes/follow-paths.sh \ + flakes/bundle.sh \ + flakes/check.sh \ + flakes/unlocked-override.sh \ + flakes/absolute-paths.sh \ + flakes/build-paths.sh \ + flakes/flake-in-submodule.sh \ ca/gc.sh \ gc.sh \ remote-store.sh \ + legacy-ssh-store.sh \ lang.sh \ + experimental-features.sh \ fetchMercurial.sh \ gc-auto.sh \ user-envs.sh \ + user-envs-migration.sh \ binary-cache.sh \ multiple-outputs.sh \ ca/build.sh \ + ca/new-build-cmd.sh \ nix-build.sh \ gc-concurrent.sh \ repair.sh \ @@ -23,6 +40,7 @@ fetchGit.sh \ fetchurl.sh \ fetchPath.sh \ + fetchTree-file.sh \ simple.sh \ referrers.sh \ optimise-store.sh \ @@ -43,7 +61,7 @@ secure-drv-outputs.sh \ restricted.sh \ fetchGitSubmodules.sh \ - flake-searching.sh \ + flakes/search-root.sh \ ca/duplicate-realisation-in-closure.sh \ readfile-context.sh \ nix-channel.sh \ @@ -52,6 +70,11 @@ check-reqs.sh \ build-remote-content-addressed-fixed.sh \ build-remote-content-addressed-floating.sh \ + build-remote-trustless-should-pass-0.sh \ + build-remote-trustless-should-pass-1.sh \ + build-remote-trustless-should-pass-2.sh \ + build-remote-trustless-should-pass-3.sh \ + build-remote-trustless-should-fail-0.sh \ nar-access.sh \ pure-eval.sh \ eval.sh \ @@ -79,37 +102,58 @@ nix-copy-ssh.sh \ post-hook.sh \ function-trace.sh \ - flake-local-settings.sh \ + flakes/config.sh \ fmt.sh \ eval-store.sh \ why-depends.sh \ + ca/why-depends.sh \ + derivation-json.sh \ + ca/derivation-json.sh \ import-derivation.sh \ ca/import-derivation.sh \ + dyn-drv/text-hashed-output.sh \ + dyn-drv/recursive-mod-json.sh \ nix_path.sh \ case-hack.sh \ placeholders.sh \ ssh-relay.sh \ plugins.sh \ build.sh \ + build-delete.sh \ + output-normalization.sh \ ca/nix-run.sh \ + selfref-gc.sh ca/selfref-gc.sh \ db-migration.sh \ bash-profile.sh \ pass-as-file.sh \ - describe-stores.sh \ nix-profile.sh \ suggestions.sh \ store-ping.sh \ fetchClosure.sh \ - impure-derivations.sh + completions.sh \ + flakes/show.sh \ + impure-derivations.sh \ + path-from-hash-part.sh \ + toString-path.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh endif -install-tests += $(foreach x, $(nix_tests), tests/$(x)) +install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) -tests-environment = NIX_REMOTE= $(bash) -e +clean-files += \ + $(d)/common/vars-and-functions.sh \ + $(d)/config.nix \ + $(d)/ca/config.nix \ + $(d)/dyn-drv/config.nix -clean-files += $(d)/common.sh $(d)/config.nix $(d)/ca/config.nix +test-deps += \ + tests/common/vars-and-functions.sh \ + tests/config.nix \ + tests/ca/config.nix \ + tests/dyn-drv/config.nix -test-deps += tests/common.sh tests/config.nix tests/ca/config.nix tests/plugins/libplugintest.$(SO_EXT) +ifeq ($(BUILD_SHARED_LIBS), 1) + test-deps += tests/plugins/libplugintest.$(SO_EXT) +endif diff -Nru nix-2.8.0/tests/local-store.sh nix-2.16.1+dfsg/tests/local-store.sh --- nix-2.8.0/tests/local-store.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/local-store.sh 2023-06-06 10:55:23.000000000 +0000 @@ -17,3 +17,6 @@ PATH3=$(nix path-info --store "local?root=$PWD/x" $CORRECT_PATH) [ $CORRECT_PATH == $PATH3 ] + +# Ensure store ping trusted works with local store +nix --store ./x store ping --json | jq -e '.trusted' diff -Nru nix-2.8.0/tests/misc.sh nix-2.16.1+dfsg/tests/misc.sh --- nix-2.8.0/tests/misc.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/misc.sh 2023-06-06 10:55:23.000000000 +0000 @@ -3,17 +3,17 @@ # Tests miscellaneous commands. # Do all commands have help? -#nix-env --help | grep -q install -#nix-store --help | grep -q realise -#nix-instantiate --help | grep -q eval -#nix-hash --help | grep -q base32 +#nix-env --help | grepQuiet install +#nix-store --help | grepQuiet realise +#nix-instantiate --help | grepQuiet eval +#nix-hash --help | grepQuiet base32 # Can we ask for the version number? nix-env --version | grep "$version" # Usage errors. -nix-env --foo 2>&1 | grep "no operation" -nix-env -q --foo 2>&1 | grep "unknown flag" +expect 1 nix-env --foo 2>&1 | grep "no operation" +expect 1 nix-env -q --foo 2>&1 | grep "unknown flag" # Eval Errors. eval_arg_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true) diff -Nru nix-2.8.0/tests/multiple-outputs.nix nix-2.16.1+dfsg/tests/multiple-outputs.nix --- nix-2.8.0/tests/multiple-outputs.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/multiple-outputs.nix 2023-06-06 10:55:23.000000000 +0000 @@ -31,6 +31,15 @@ helloString = "Hello, world!"; }; + use-a = mkDerivation { + name = "use-a"; + inherit (a) first second; + builder = builtins.toFile "builder.sh" + '' + cat $first/file $second/file >$out + ''; + }; + b = mkDerivation { defaultOutput = assert a.second.helloString == "Hello, world!"; a; firstOutput = assert a.outputName == "first"; a.first.first; @@ -80,4 +89,42 @@ ''; }).a; + e = mkDerivation { + name = "multiple-outputs-e"; + outputs = [ "a_a" "b" "c" ]; + meta.outputsToInstall = [ "a_a" "b" ]; + buildCommand = "mkdir $a_a $b $c"; + }; + + independent = mkDerivation { + name = "multiple-outputs-independent"; + outputs = [ "first" "second" ]; + builder = builtins.toFile "builder.sh" + '' + mkdir $first $second + test -z $all + echo "first" > $first/file + echo "second" > $second/file + ''; + }; + + use-independent = mkDerivation { + name = "use-independent"; + inherit (a) first second; + builder = builtins.toFile "builder.sh" + '' + cat $first/file $second/file >$out + ''; + }; + + invalid-output-name-1 = mkDerivation { + name = "invalid-output-name-1"; + outputs = [ "out/"]; + }; + + invalid-output-name-2 = mkDerivation { + name = "invalid-output-name-2"; + outputs = [ "x" "foo$"]; + }; + } diff -Nru nix-2.8.0/tests/multiple-outputs.sh nix-2.16.1+dfsg/tests/multiple-outputs.sh --- nix-2.8.0/tests/multiple-outputs.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/multiple-outputs.sh 2023-06-06 10:55:23.000000000 +0000 @@ -19,8 +19,8 @@ # outputs. drvPath=$(nix-instantiate multiple-outputs.nix -A c) #[ "$drvPath" = "$drvPath2" ] -grep -q 'multiple-outputs-a.drv",\["first","second"\]' $drvPath -grep -q 'multiple-outputs-b.drv",\["out"\]' $drvPath +grepQuiet 'multiple-outputs-a.drv",\["first","second"\]' $drvPath +grepQuiet 'multiple-outputs-b.drv",\["out"\]' $drvPath # While we're at it, test the ‘unsafeDiscardOutputDependency’ primop. outPath=$(nix-build multiple-outputs.nix -A d --no-out-link) @@ -83,3 +83,6 @@ nix-store --gc --print-roots rm -rf $NIX_STORE_DIR/.links rmdir $NIX_STORE_DIR + +expect 1 nix build -f multiple-outputs.nix invalid-output-name-1 2>&1 | grep 'contains illegal character' +expect 1 nix build -f multiple-outputs.nix invalid-output-name-2 2>&1 | grep 'contains illegal character' diff -Nru nix-2.8.0/tests/nar-access.sh nix-2.16.1+dfsg/tests/nar-access.sh --- nix-2.8.0/tests/nar-access.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nar-access.sh 2023-06-06 10:55:23.000000000 +0000 @@ -46,8 +46,8 @@ <(echo '{"type":"regular","size":0}' | jq -S) # Test missing files. -nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' -nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' +expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' +expect 1 nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' # Test failure to dump. if nix-store --dump $storePath >/dev/full ; then diff -Nru nix-2.8.0/tests/nix-channel.sh nix-2.16.1+dfsg/tests/nix-channel.sh --- nix-2.8.0/tests/nix-channel.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nix-channel.sh 2023-06-06 10:55:23.000000000 +0000 @@ -6,12 +6,25 @@ # Test add/list/remove. nix-channel --add http://foo/bar xyzzy -nix-channel --list | grep -q http://foo/bar +nix-channel --list | grepQuiet http://foo/bar nix-channel --remove xyzzy [ -e $TEST_HOME/.nix-channels ] [ "$(cat $TEST_HOME/.nix-channels)" = '' ] +# Test the XDG Base Directories support + +export NIX_CONFIG="use-xdg-base-directories = true" + +nix-channel --add http://foo/bar xyzzy +nix-channel --list | grepQuiet http://foo/bar +nix-channel --remove xyzzy + +unset NIX_CONFIG + +[ -e $TEST_HOME/.local/state/nix/channels ] +[ "$(cat $TEST_HOME/.local/state/nix/channels)" = '' ] + # Create a channel. rm -rf $TEST_ROOT/foo mkdir -p $TEST_ROOT/foo @@ -28,8 +41,8 @@ # Do a query. nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml -grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml -grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml +grepQuiet 'meta.*description.*Random test package' $TEST_ROOT/meta.xml +grepQuiet 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml # Do an install. nix-env -i dependencies-top @@ -41,9 +54,9 @@ # Do a query. nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml -grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml -grep -q 'item.*attrPath="bar".*name="dependencies-top"' $TEST_ROOT/meta.xml -grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml +grepQuiet 'meta.*description.*Random test package' $TEST_ROOT/meta.xml +grepQuiet 'item.*attrPath="bar".*name="dependencies-top"' $TEST_ROOT/meta.xml +grepQuiet 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml # Do an install. nix-env -i dependencies-top diff -Nru nix-2.8.0/tests/nix-copy-closure.nix nix-2.16.1+dfsg/tests/nix-copy-closure.nix --- nix-2.8.0/tests/nix-copy-closure.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nix-copy-closure.nix 1970-01-01 00:00:00.000000000 +0000 @@ -1,77 +0,0 @@ -# Test ‘nix-copy-closure’. - -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - -makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pkgs.tmux; in { - name = "nix-copy-closure"; - - nodes = - { client = - { config, lib, pkgs, ... }: - { virtualisation.writableStore = true; - virtualisation.pathsInNixDB = [ pkgA pkgD.drvPath ]; - nix.binaryCaches = lib.mkForce [ ]; - }; - - server = - { config, pkgs, ... }: - { services.openssh.enable = true; - virtualisation.writableStore = true; - virtualisation.pathsInNixDB = [ pkgB pkgC ]; - }; - }; - - testScript = { nodes }: '' - # fmt: off - import subprocess - - start_all() - - # Create an SSH key on the client. - subprocess.run([ - "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" - ], capture_output=True, check=True) - - client.succeed("mkdir -m 700 /root/.ssh") - client.copy_from_host("key", "/root/.ssh/id_ed25519") - client.succeed("chmod 600 /root/.ssh/id_ed25519") - - # Install the SSH key on the server. - server.succeed("mkdir -m 700 /root/.ssh") - server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") - server.wait_for_unit("sshd") - client.wait_for_unit("network.target") - client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") - - # Copy the closure of package A from the client to the server. - server.fail("nix-store --check-validity ${pkgA}") - client.succeed("nix-copy-closure --to server --gzip ${pkgA} >&2") - server.succeed("nix-store --check-validity ${pkgA}") - - # Copy the closure of package B from the server to the client. - client.fail("nix-store --check-validity ${pkgB}") - client.succeed("nix-copy-closure --from server --gzip ${pkgB} >&2") - client.succeed("nix-store --check-validity ${pkgB}") - - # Copy the closure of package C via the SSH substituter. - client.fail("nix-store -r ${pkgC}") - - # Copy the derivation of package D's derivation from the client to the server. - server.fail("nix-store --check-validity ${pkgD.drvPath}") - client.succeed("nix-copy-closure --to server --gzip ${pkgD.drvPath} >&2") - server.succeed("nix-store --check-validity ${pkgD.drvPath}") - - # FIXME - # client.succeed( - # "nix-store --option use-ssh-substituter true" - # " --option ssh-substituter-hosts root\@server" - # " -r ${pkgC} >&2" - # ) - # client.succeed("nix-store --check-validity ${pkgC}") - ''; -}) diff -Nru nix-2.8.0/tests/nix-daemon-untrusting.sh nix-2.16.1+dfsg/tests/nix-daemon-untrusting.sh --- nix-2.8.0/tests/nix-daemon-untrusting.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nix-daemon-untrusting.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,3 @@ +#!/bin/sh + +exec nix-daemon --force-untrusted "$@" diff -Nru nix-2.8.0/tests/nixos/authorization.nix nix-2.16.1+dfsg/tests/nixos/authorization.nix --- nix-2.8.0/tests/nixos/authorization.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nixos/authorization.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,79 @@ +{ + name = "authorization"; + + nodes.machine = { + virtualisation.writableStore = true; + # TODO add a test without allowed-users setting. allowed-users is uncommon among NixOS users. + nix.settings.allowed-users = ["alice" "bob"]; + nix.settings.trusted-users = ["alice"]; + + users.users.alice.isNormalUser = true; + users.users.bob.isNormalUser = true; + users.users.mallory.isNormalUser = true; + + nix.settings.experimental-features = "nix-command"; + }; + + testScript = + let + pathFour = "/nix/store/20xfy868aiic0r0flgzq4n5dq1yvmxkn-four"; + in + '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + exec 1>&2 + echo kSELDhobKaF8/VdxIxdP7EQe+Q > one + diff $(nix store add-file one) one + """) + machine.succeed(""" + su --login alice -c ' + set -x + cd ~ + echo ehHtmfuULXYyBV6NBk6QUi8iE0 > two + ls + diff $(echo $(nix store add-file two)) two' 1>&2 + """) + machine.succeed(""" + su --login bob -c ' + set -x + cd ~ + echo 0Jw8RNp7cK0W2AdNbcquofcOVk > three + diff $(nix store add-file three) three + ' 1>&2 + """) + + # We're going to check that a path is not created + machine.succeed(""" + ! [[ -e ${pathFour} ]] + """) + machine.succeed(""" + su --login mallory -c ' + set -x + cd ~ + echo 5mgtDj0ohrWkT50TLR0f4tIIxY > four; + (! nix store add-file four 2>&1) | grep -F "cannot open connection to remote store" + (! nix store add-file four 2>&1) | grep -F "Connection reset by peer" + ! [[ -e ${pathFour} ]] + ' 1>&2 + """) + + # Check that the file _can_ be added, and matches the expected path we were checking + machine.succeed(""" + exec 1>&2 + echo 5mgtDj0ohrWkT50TLR0f4tIIxY > four + four="$(nix store add-file four)" + diff $four four + diff <(echo $four) <(echo ${pathFour}) + """) + + machine.succeed(""" + su --login alice -c 'nix-store --verify --repair' + """) + + machine.succeed(""" + set -x + su --login bob -c '(! nix-store --verify --repair 2>&1)' | tee diag 1>&2 + grep -F "you are not privileged to repair paths" diag + """) + ''; +} diff -Nru nix-2.8.0/tests/nixos/containers/containers.nix nix-2.16.1+dfsg/tests/nixos/containers/containers.nix --- nix-2.8.0/tests/nixos/containers/containers.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nixos/containers/containers.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,63 @@ +# Test whether we can run a NixOS container inside a Nix build using systemd-nspawn. +{ lib, nixpkgs, ... }: + +{ + name = "containers"; + + nodes = + { + host = + { config, lib, pkgs, nodes, ... }: + { virtualisation.writableStore = true; + virtualisation.diskSize = 2048; + virtualisation.additionalPaths = + [ pkgs.stdenvNoCC + (import ./systemd-nspawn.nix { inherit nixpkgs; }).toplevel + ]; + virtualisation.memorySize = 4096; + nix.settings.substituters = lib.mkForce [ ]; + nix.extraOptions = + '' + extra-experimental-features = nix-command auto-allocate-uids cgroups + extra-system-features = uid-range + ''; + nix.nixPath = [ "nixpkgs=${nixpkgs}" ]; + }; + }; + + testScript = { nodes }: '' + start_all() + + host.succeed("nix --version >&2") + + # Test that 'id' gives the expected result in various configurations. + + # Existing UIDs, sandbox. + host.succeed("nix build -v --no-auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-1") + host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]") + + # Existing UIDs, no sandbox. + host.succeed("nix build -v --no-auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-2") + host.succeed("[[ $(cat ./result) = 'uid=30001(nixbld1) gid=30000(nixbld) groups=30000(nixbld)' ]]") + + # Auto-allocated UIDs, sandbox. + host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-3") + host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]") + + # Auto-allocated UIDs, no sandbox. + host.succeed("nix build -v --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-4") + host.succeed("[[ $(cat ./result) = 'uid=872415232 gid=30000(nixbld) groups=30000(nixbld)' ]]") + + # Auto-allocated UIDs, UID range, sandbox. + host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-5 --arg uidRange true") + host.succeed("[[ $(cat ./result) = 'uid=0(root) gid=0(root) groups=0(root)' ]]") + + # Auto-allocated UIDs, UID range, no sandbox. + host.fail("nix build -v --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true") + + # Run systemd-nspawn in a Nix build. + host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") + host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") + ''; + +} diff -Nru nix-2.8.0/tests/nixos/containers/id-test.nix nix-2.16.1+dfsg/tests/nixos/containers/id-test.nix --- nix-2.8.0/tests/nixos/containers/id-test.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nixos/containers/id-test.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,8 @@ +{ name, uidRange ? false }: + +with import {}; + +runCommand name + { requiredSystemFeatures = if uidRange then ["uid-range"] else []; + } + "id; id > $out" diff -Nru nix-2.8.0/tests/nixos/containers/systemd-nspawn.nix nix-2.16.1+dfsg/tests/nixos/containers/systemd-nspawn.nix --- nix-2.8.0/tests/nixos/containers/systemd-nspawn.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nixos/containers/systemd-nspawn.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,78 @@ +{ nixpkgs }: + +let + + machine = { config, pkgs, ... }: + { + system.stateVersion = "22.05"; + boot.isContainer = true; + systemd.services.console-getty.enable = false; + networking.dhcpcd.enable = false; + + services.httpd = { + enable = true; + adminAddr = "nixos@example.org"; + }; + + systemd.services.test = { + wantedBy = [ "multi-user.target" ]; + after = [ "httpd.service" ]; + script = '' + source /.env + echo "Hello World" > $out/msg + ls -lR /dev > $out/dev + ${pkgs.curl}/bin/curl -sS --fail http://localhost/ > $out/page.html + ''; + unitConfig = { + FailureAction = "exit-force"; + FailureActionExitStatus = 42; + SuccessAction = "exit-force"; + }; + }; + }; + + cfg = (import (nixpkgs + "/nixos/lib/eval-config.nix") { + modules = [ machine ]; + system = "x86_64-linux"; + }); + + config = cfg.config; + +in + +with cfg._module.args.pkgs; + +runCommand "test" + { buildInputs = [ config.system.path ]; + requiredSystemFeatures = [ "uid-range" ]; + toplevel = config.system.build.toplevel; + } + '' + root=$(pwd)/root + mkdir -p $root $root/etc + + export > $root/.env + + # Make /run a tmpfs to shut up a systemd warning. + mkdir /run + mount -t tmpfs none /run + + mount -t cgroup2 none /sys/fs/cgroup + + mkdir -p $out + + chmod +w /etc + touch /etc/os-release + echo a5ea3f98dedc0278b6f3cc8c37eeaeac > /etc/machine-id + + SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=1 \ + ${config.systemd.package}/bin/systemd-nspawn \ + --keep-unit \ + -M ${config.networking.hostName} -D "$root" \ + --register=no \ + --resolv-conf=off \ + --bind-ro=/nix/store \ + --bind=$out \ + --private-network \ + $toplevel/init + '' diff -Nru nix-2.8.0/tests/nixos/github-flakes.nix nix-2.16.1+dfsg/tests/nixos/github-flakes.nix --- nix-2.8.0/tests/nixos/github-flakes.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nixos/github-flakes.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,203 @@ +{ lib, config, nixpkgs, ... }: +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + # Generate a fake root CA and a fake api.github.com / github.com / channels.nixos.org certificate. + cert = pkgs.runCommand "cert" { nativeBuildInputs = [ pkgs.openssl ]; } + '' + mkdir -p $out + + openssl genrsa -out ca.key 2048 + openssl req -new -x509 -days 36500 -key ca.key \ + -subj "/C=NL/ST=Denial/L=Springfield/O=Dis/CN=Root CA" -out $out/ca.crt + + openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \ + -subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=github.com" -out server.csr + openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:github.com,DNS:channels.nixos.org") \ + -days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt + ''; + + registry = pkgs.writeTextFile { + name = "registry"; + text = '' + { + "flakes": [ + { + "from": { + "type": "indirect", + "id": "nixpkgs" + }, + "to": { + "type": "github", + "owner": "NixOS", + "repo": "nixpkgs" + } + }, + { + "from": { + "type": "indirect", + "id": "private-flake" + }, + "to": { + "type": "github", + "owner": "fancy-enterprise", + "repo": "private-flake" + } + } + ], + "version": 2 + } + ''; + destination = "/flake-registry.json"; + }; + + private-flake-rev = "9f1dd0df5b54a7dc75b618034482ed42ce34383d"; + + private-flake-api = pkgs.runCommand "private-flake" {} + '' + mkdir -p $out/{commits,tarball} + + # Setup https://docs.github.com/en/rest/commits/commits#get-a-commit + echo '{"sha": "${private-flake-rev}"}' > $out/commits/HEAD + + # Setup tarball download via API + dir=private-flake + mkdir $dir + echo '{ outputs = {...}: {}; }' > $dir/flake.nix + tar cfz $out/tarball/${private-flake-rev} $dir --hard-dereference + ''; + + nixpkgs-api = pkgs.runCommand "nixpkgs-flake" {} + '' + mkdir -p $out/commits + + # Setup https://docs.github.com/en/rest/commits/commits#get-a-commit + echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD + ''; + + archive = pkgs.runCommand "nixpkgs-flake" {} + '' + mkdir -p $out/archive + + dir=NixOS-nixpkgs-${nixpkgs.shortRev} + cp -prd ${nixpkgs} $dir + # Set the correct timestamp in the tarball. + find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference + ''; +in + +{ + name = "github-flakes"; + + nodes = + { + github = + { config, pkgs, ... }: + { networking.firewall.allowedTCPPorts = [ 80 443 ]; + + services.httpd.enable = true; + services.httpd.adminAddr = "foo@example.org"; + services.httpd.extraConfig = '' + ErrorLog syslog:local6 + ''; + services.httpd.virtualHosts."channels.nixos.org" = + { forceSSL = true; + sslServerKey = "${cert}/server.key"; + sslServerCert = "${cert}/server.crt"; + servedDirs = + [ { urlPath = "/"; + dir = registry; + } + ]; + }; + services.httpd.virtualHosts."api.github.com" = + { forceSSL = true; + sslServerKey = "${cert}/server.key"; + sslServerCert = "${cert}/server.crt"; + servedDirs = + [ { urlPath = "/repos/NixOS/nixpkgs"; + dir = nixpkgs-api; + } + { urlPath = "/repos/fancy-enterprise/private-flake"; + dir = private-flake-api; + } + ]; + }; + services.httpd.virtualHosts."github.com" = + { forceSSL = true; + sslServerKey = "${cert}/server.key"; + sslServerCert = "${cert}/server.crt"; + servedDirs = + [ { urlPath = "/NixOS/nixpkgs"; + dir = archive; + } + ]; + }; + }; + + client = + { config, lib, pkgs, nodes, ... }: + { virtualisation.writableStore = true; + virtualisation.diskSize = 2048; + virtualisation.additionalPaths = [ pkgs.hello pkgs.fuse ]; + virtualisation.memorySize = 4096; + nix.settings.substituters = lib.mkForce [ ]; + nix.extraOptions = "experimental-features = nix-command flakes"; + networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} = + [ "channels.nixos.org" "api.github.com" "github.com" ]; + security.pki.certificateFiles = [ "${cert}/ca.crt" ]; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import json + import time + + start_all() + + def cat_log(): + github.succeed("cat /var/log/httpd/*.log >&2") + + github.wait_for_unit("httpd.service") + + client.succeed("curl -v https://github.com/ >&2") + out = client.succeed("nix registry list") + print(out) + assert "github:NixOS/nixpkgs" in out, "nixpkgs flake not found" + assert "github:fancy-enterprise/private-flake" in out, "private flake not found" + cat_log() + + # If no github access token is provided, nix should use the public archive url... + out = client.succeed("nix flake metadata nixpkgs --json") + print(out) + info = json.loads(out) + assert info["revision"] == "${nixpkgs.rev}", f"revision mismatch: {info['revision']} != ${nixpkgs.rev}" + cat_log() + + # ... otherwise it should use the API + out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0") + print(out) + info = json.loads(out) + assert info["revision"] == "${private-flake-rev}", f"revision mismatch: {info['revision']} != ${private-flake-rev}" + cat_log() + + client.succeed("nix registry pin nixpkgs") + client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2") + + # Shut down the web server. The flake should be cached on the client. + github.succeed("systemctl stop httpd.service") + + info = json.loads(client.succeed("nix flake metadata nixpkgs --json")) + date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified'])) + assert date == "${nixpkgs.lastModifiedDate}", "time mismatch" + + client.succeed("nix build nixpkgs#hello") + + # The build shouldn't fail even with --tarball-ttl 0 (the server + # being down should not be a fatal error). + client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") + ''; + +} diff -Nru nix-2.8.0/tests/nixos/nix-copy-closure.nix nix-2.16.1+dfsg/tests/nixos/nix-copy-closure.nix --- nix-2.8.0/tests/nixos/nix-copy-closure.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nixos/nix-copy-closure.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,80 @@ +# Test ‘nix-copy-closure’. + +{ lib, config, nixpkgs, hostPkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + pkgA = pkgs.cowsay; + pkgB = pkgs.wget; + pkgC = pkgs.hello; + pkgD = pkgs.tmux; + +in { + name = "nix-copy-closure"; + + nodes = + { client = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgA pkgD.drvPath ]; + nix.settings.substituters = lib.mkForce [ ]; + }; + + server = + { config, pkgs, ... }: + { services.openssh.enable = true; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgB pkgC ]; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + start_all() + + # Create an SSH key on the client. + subprocess.run([ + "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + + client.succeed("mkdir -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the server. + server.succeed("mkdir -m 700 /root/.ssh") + server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + server.wait_for_unit("sshd") + client.wait_for_unit("network.target") + client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") + + # Copy the closure of package A from the client to the server. + server.fail("nix-store --check-validity ${pkgA}") + client.succeed("nix-copy-closure --to server --gzip ${pkgA} >&2") + server.succeed("nix-store --check-validity ${pkgA}") + + # Copy the closure of package B from the server to the client. + client.fail("nix-store --check-validity ${pkgB}") + client.succeed("nix-copy-closure --from server --gzip ${pkgB} >&2") + client.succeed("nix-store --check-validity ${pkgB}") + + # Copy the closure of package C via the SSH substituter. + client.fail("nix-store -r ${pkgC}") + + # Copy the derivation of package D's derivation from the client to the server. + server.fail("nix-store --check-validity ${pkgD.drvPath}") + client.succeed("nix-copy-closure --to server --gzip ${pkgD.drvPath} >&2") + server.succeed("nix-store --check-validity ${pkgD.drvPath}") + + # FIXME + # client.succeed( + # "nix-store --option use-ssh-substituter true" + # " --option ssh-substituter-hosts root\@server" + # " -r ${pkgC} >&2" + # ) + # client.succeed("nix-store --check-validity ${pkgC}") + ''; +} diff -Nru nix-2.8.0/tests/nixos/nix-copy.nix nix-2.16.1+dfsg/tests/nixos/nix-copy.nix --- nix-2.8.0/tests/nixos/nix-copy.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nixos/nix-copy.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,95 @@ +# Test that ‘nix copy’ works over ssh. + +{ lib, config, nixpkgs, hostPkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + pkgA = pkgs.cowsay; + pkgB = pkgs.wget; + pkgC = pkgs.hello; + pkgD = pkgs.tmux; + +in { + name = "nix-copy"; + + enableOCR = true; + + nodes = + { client = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgA pkgD.drvPath ]; + nix.settings.substituters = lib.mkForce [ ]; + nix.settings.experimental-features = [ "nix-command" ]; + services.getty.autologinUser = "root"; + programs.ssh.extraConfig = '' + Host * + ControlMaster auto + ControlPath ~/.ssh/master-%h:%r@%n:%p + ControlPersist 15m + ''; + }; + + server = + { config, pkgs, ... }: + { services.openssh.enable = true; + services.openssh.permitRootLogin = "yes"; + users.users.root.password = "foobar"; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgB pkgC ]; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + # Create an SSH key on the client. + subprocess.run([ + "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + + start_all() + + server.wait_for_unit("sshd") + client.wait_for_unit("network.target") + client.wait_for_unit("getty@tty1.service") + client.wait_for_text("]#") + + # Copy the closure of package A from the client to the server using password authentication, + # and check that all prompts are visible + server.fail("nix-store --check-validity ${pkgA}") + client.send_chars("nix copy --to ssh://server ${pkgA} >&2; echo done\n") + client.wait_for_text("continue connecting") + client.send_chars("yes\n") + client.wait_for_text("Password:") + client.send_chars("foobar\n") + client.wait_for_text("done") + server.succeed("nix-store --check-validity ${pkgA}") + + # Check that ControlMaster is working + client.send_chars("nix copy --to ssh://server ${pkgA} >&2; echo done\n") + client.wait_for_text("done") + + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the server. + server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + server.succeed("systemctl restart sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") + + # Copy the closure of package B from the server to the client, using ssh-ng. + client.fail("nix-store --check-validity ${pkgB}") + # Shouldn't download untrusted paths by default + client.fail("nix copy --from ssh-ng://server ${pkgB} >&2") + client.succeed("nix copy --no-check-sigs --from ssh-ng://server ${pkgB} >&2") + client.succeed("nix-store --check-validity ${pkgB}") + + # Copy the derivation of package D's derivation from the client to the server. + server.fail("nix-store --check-validity ${pkgD.drvPath}") + client.succeed("nix copy --derivation --to ssh://server ${pkgD.drvPath} >&2") + server.succeed("nix-store --check-validity ${pkgD.drvPath}") + ''; +} diff -Nru nix-2.8.0/tests/nixos/nss-preload.nix nix-2.16.1+dfsg/tests/nixos/nss-preload.nix --- nix-2.8.0/tests/nixos/nss-preload.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nixos/nss-preload.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,121 @@ +{ lib, config, nixpkgs, ... }: + +let + + pkgs = config.nodes.client.nixpkgs.pkgs; + + nix-fetch = pkgs.writeText "fetch.nix" '' + derivation { + # This derivation is an copy from what is available over at + # nix.git:corepkgs/fetchurl.nix + builder = "builtin:fetchurl"; + + # We're going to fetch data from the http_dns instance created before + # we expect the content to be the same as the content available there. + # ``` + # $ nix-hash --type sha256 --to-base32 $(echo "hello world" | sha256sum | cut -d " " -f 1) + # 0ix4jahrkll5zg01wandq78jw3ab30q4nscph67rniqg5x7r0j59 + # ``` + outputHash = "0ix4jahrkll5zg01wandq78jw3ab30q4nscph67rniqg5x7r0j59"; + outputHashAlgo = "sha256"; + outputHashMode = "flat"; + + name = "example.com"; + url = "http://example.com"; + + unpack = false; + executable = false; + + system = "builtin"; + + preferLocalBuild = true; + + impureEnvVars = [ + "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" + ]; + + urls = [ "http://example.com" ]; + } + ''; +in + +{ + name = "nss-preload"; + + nodes = { + http_dns = { lib, pkgs, config, ... }: { + networking.firewall.enable = false; + networking.interfaces.eth1.ipv6.addresses = lib.mkForce [ + { address = "fd21::1"; prefixLength = 64; } + ]; + networking.interfaces.eth1.ipv4.addresses = lib.mkForce [ + { address = "192.168.0.1"; prefixLength = 24; } + ]; + + services.unbound = { + enable = true; + enableRootTrustAnchor = false; + settings = { + server = { + interface = [ "192.168.0.1" "fd21::1" "::1" "127.0.0.1" ]; + access-control = [ "192.168.0.0/24 allow" "fd21::/64 allow" "::1 allow" "127.0.0.0/8 allow" ]; + local-data = [ + ''"example.com. IN A 192.168.0.1"'' + ''"example.com. IN AAAA fd21::1"'' + ''"tarballs.nixos.org. IN A 192.168.0.1"'' + ''"tarballs.nixos.org. IN AAAA fd21::1"'' + ]; + }; + }; + }; + + services.nginx = { + enable = true; + virtualHosts."example.com" = { + root = pkgs.runCommand "testdir" {} '' + mkdir "$out" + echo hello world > "$out/index.html" + ''; + }; + }; + }; + + # client consumes a remote resolver + client = { lib, nodes, pkgs, ... }: { + networking.useDHCP = false; + networking.nameservers = [ + (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv6.addresses).address + (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv4.addresses).address + ]; + networking.interfaces.eth1.ipv6.addresses = [ + { address = "fd21::10"; prefixLength = 64; } + ]; + networking.interfaces.eth1.ipv4.addresses = [ + { address = "192.168.0.10"; prefixLength = 24; } + ]; + + nix.settings.extra-sandbox-paths = lib.mkForce []; + nix.settings.substituters = lib.mkForce []; + nix.settings.sandbox = lib.mkForce true; + }; + }; + + testScript = { nodes, ... }: '' + http_dns.wait_for_unit("nginx") + http_dns.wait_for_open_port(80) + http_dns.wait_for_unit("unbound") + http_dns.wait_for_open_port(53) + + client.start() + client.wait_for_unit('multi-user.target') + + with subtest("can fetch data from a remote server outside sandbox"): + client.succeed("nix --version >&2") + client.succeed("curl -vvv http://example.com/index.html >&2") + + with subtest("nix-build can lookup dns and fetch data"): + client.succeed(""" + nix-build ${nix-fetch} >&2 + """) + ''; +} diff -Nru nix-2.8.0/tests/nixos/remote-builds.nix nix-2.16.1+dfsg/tests/nixos/remote-builds.nix --- nix-2.8.0/tests/nixos/remote-builds.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nixos/remote-builds.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,111 @@ +# Test Nix's remote build feature. + +{ config, lib, hostPkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + # The configuration of the remote builders. + builder = + { config, pkgs, ... }: + { services.openssh.enable = true; + virtualisation.writableStore = true; + nix.settings.sandbox = true; + + # Regression test for use of PID namespaces when /proc has + # filesystems mounted on top of it + # (i.e. /proc/sys/fs/binfmt_misc). + boot.binfmt.emulatedSystems = [ "aarch64-linux" ]; + }; + + # Trivial Nix expression to build remotely. + expr = config: nr: pkgs.writeText "expr.nix" + '' + let utils = builtins.storePath ${config.system.build.extraUtils}; in + derivation { + name = "hello-${toString nr}"; + system = "i686-linux"; + PATH = "''${utils}/bin"; + builder = "''${utils}/bin/sh"; + args = [ "-c" "if [ ${toString nr} = 5 ]; then echo FAIL; exit 1; fi; echo Hello; mkdir $out $foo; cat /proc/sys/kernel/hostname > $out/host; ln -s $out $foo/bar; sleep 10" ]; + outputs = [ "out" "foo" ]; + } + ''; + +in + +{ + name = "remote-builds"; + + nodes = + { builder1 = builder; + builder2 = builder; + + client = + { config, lib, pkgs, ... }: + { nix.settings.max-jobs = 0; # force remote building + nix.distributedBuilds = true; + nix.buildMachines = + [ { hostName = "builder1"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + } + { hostName = "builder2"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + } + ]; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ config.system.build.extraUtils ]; + nix.settings.substituters = lib.mkForce [ ]; + programs.ssh.extraConfig = "ConnectTimeout 30"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + start_all() + + # Create an SSH key on the client. + subprocess.run([ + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the builders. + client.wait_for_unit("network.target") + for builder in [builder1, builder2]: + builder.succeed("mkdir -p -m 700 /root/.ssh") + builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + builder.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + + # Perform a build and check that it was performed on the builder. + out = client.succeed( + "nix-build ${expr nodes.client.config 1} 2> build-output", + "grep -q Hello build-output" + ) + builder1.succeed(f"test -e {out}") + + # And a parallel build. + paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out') + out1, out2 = paths.split() + builder1.succeed(f"test -e {out1} -o -e {out2}") + builder2.succeed(f"test -e {out1} -o -e {out2}") + + # And a failing build. + client.fail("nix-build ${expr nodes.client.config 5}") + + # Test whether the build hook automatically skips unavailable builders. + builder1.block() + client.succeed("nix-build ${expr nodes.client.config 4}") + ''; +} diff -Nru nix-2.8.0/tests/nixos/setuid.nix nix-2.16.1+dfsg/tests/nixos/setuid.nix --- nix-2.8.0/tests/nixos/setuid.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nixos/setuid.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,125 @@ +# Verify that Linux builds cannot create setuid or setgid binaries. + +{ lib, config, nixpkgs, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + +in +{ + name = "setuid"; + + nodes.machine = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + nix.settings.substituters = lib.mkForce [ ]; + nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ]; + virtualisation.additionalPaths = [ pkgs.stdenvNoCC pkgs.pkgsi686Linux.stdenvNoCC ]; + }; + + testScript = { nodes }: '' + # fmt: off + start_all() + + # Copying to /tmp should succeed. + machine.succeed(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + ")' + """.strip()) + + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') + + machine.succeed("rm /tmp/id") + + # Creating a setuid binary should fail. + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 4755 /tmp/id + ")' + """.strip()) + + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') + + machine.succeed("rm /tmp/id") + + # Creating a setgid binary should fail. + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 2755 /tmp/id + ")' + """.strip()) + + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') + + machine.succeed("rm /tmp/id") + + # The checks should also work on 32-bit binaries. + machine.fail(r""" + nix-build --no-sandbox -E '(with import { system = "i686-linux"; }; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 2755 /tmp/id + ")' + """.strip()) + + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') + + machine.succeed("rm /tmp/id") + + # The tests above use fchmodat(). Test chmod() as well. + machine.succeed(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"chmod 0666, qw(/tmp/id) or die\" + ")' + """.strip()) + + machine.succeed('[[ $(stat -c %a /tmp/id) = 666 ]]') + + machine.succeed("rm /tmp/id") + + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"chmod 04755, qw(/tmp/id) or die\" + ")' + """.strip()) + + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') + + machine.succeed("rm /tmp/id") + + # And test fchmod(). + machine.succeed(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\" + ")' + """.strip()) + + machine.succeed('[[ $(stat -c %a /tmp/id) = 1750 ]]') + + machine.succeed("rm /tmp/id") + + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\" + ")' + """.strip()) + + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') + + machine.succeed("rm /tmp/id") + ''; +} diff -Nru nix-2.8.0/tests/nixos/sourcehut-flakes.nix nix-2.16.1+dfsg/tests/nixos/sourcehut-flakes.nix --- nix-2.8.0/tests/nixos/sourcehut-flakes.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nixos/sourcehut-flakes.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,161 @@ +{ lib, config, hostPkgs, nixpkgs, ... }: + +let + pkgs = config.nodes.sourcehut.nixpkgs.pkgs; + + # Generate a fake root CA and a fake git.sr.ht certificate. + cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } + '' + mkdir -p $out + + openssl genrsa -out ca.key 2048 + openssl req -new -x509 -days 36500 -key ca.key \ + -subj "/C=NL/ST=Denial/L=Springfield/O=Dis/CN=Root CA" -out $out/ca.crt + + openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \ + -subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=git.sr.ht" -out server.csr + openssl x509 -req -extfile <(printf "subjectAltName=DNS:git.sr.ht") \ + -days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt + ''; + + registry = pkgs.writeTextFile { + name = "registry"; + text = '' + { + "flakes": [ + { + "from": { + "type": "indirect", + "id": "nixpkgs" + }, + "to": { + "type": "sourcehut", + "owner": "~NixOS", + "repo": "nixpkgs" + } + } + ], + "version": 2 + } + ''; + destination = "/flake-registry.json"; + }; + + nixpkgs-repo = pkgs.runCommand "nixpkgs-flake" { } + '' + dir=NixOS-nixpkgs-${nixpkgs.shortRev} + cp -prd ${nixpkgs} $dir + + # Set the correct timestamp in the tarball. + find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + + mkdir -p $out/archive + tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference + + echo 'ref: refs/heads/master' > $out/HEAD + + mkdir -p $out/info + echo -e '${nixpkgs.rev}\trefs/heads/master\n${nixpkgs.rev}\trefs/tags/foo-bar' > $out/info/refs + ''; + +in + + { + name = "sourcehut-flakes"; + + nodes = + { + # Impersonate git.sr.ht + sourcehut = + { config, pkgs, ... }: + { + networking.firewall.allowedTCPPorts = [ 80 443 ]; + + services.httpd.enable = true; + services.httpd.adminAddr = "foo@example.org"; + services.httpd.extraConfig = '' + ErrorLog syslog:local6 + ''; + services.httpd.virtualHosts."git.sr.ht" = + { + forceSSL = true; + sslServerKey = "${cert}/server.key"; + sslServerCert = "${cert}/server.crt"; + servedDirs = + [ + { + urlPath = "/~NixOS/nixpkgs"; + dir = nixpkgs-repo; + } + { + urlPath = "/~NixOS/flake-registry/blob/master"; + dir = registry; + } + ]; + }; + }; + + client = + { config, lib, pkgs, nodes, ... }: + { + virtualisation.writableStore = true; + virtualisation.diskSize = 2048; + virtualisation.additionalPaths = [ pkgs.hello pkgs.fuse ]; + virtualisation.memorySize = 4096; + nix.settings.substituters = lib.mkForce [ ]; + nix.extraOptions = '' + experimental-features = nix-command flakes + flake-registry = https://git.sr.ht/~NixOS/flake-registry/blob/master/flake-registry.json + ''; + environment.systemPackages = [ pkgs.jq ]; + networking.hosts.${(builtins.head nodes.sourcehut.config.networking.interfaces.eth1.ipv4.addresses).address} = + [ "git.sr.ht" ]; + security.pki.certificateFiles = [ "${cert}/ca.crt" ]; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import json + import time + + start_all() + + sourcehut.wait_for_unit("httpd.service") + + client.succeed("curl -v https://git.sr.ht/ >&2") + client.succeed("nix registry list | grep nixpkgs") + + # Test that it resolves HEAD + rev = client.succeed("nix flake info sourcehut:~NixOS/nixpkgs --json | jq -r .revision") + assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" + # Test that it resolves branches + rev = client.succeed("nix flake info sourcehut:~NixOS/nixpkgs/master --json | jq -r .revision") + assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" + # Test that it resolves tags + rev = client.succeed("nix flake info sourcehut:~NixOS/nixpkgs/foo-bar --json | jq -r .revision") + assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" + + # Registry and pinning test + rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision") + assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" + + client.succeed("nix registry pin nixpkgs") + + client.succeed("nix flake info nixpkgs --tarball-ttl 0 >&2") + + # Shut down the web server. The flake should be cached on the client. + sourcehut.succeed("systemctl stop httpd.service") + + info = json.loads(client.succeed("nix flake info nixpkgs --json")) + date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified'])) + assert date == "${nixpkgs.lastModifiedDate}", "time mismatch" + + client.succeed("nix build nixpkgs#hello") + + # The build shouldn't fail even with --tarball-ttl 0 (the server + # being down should not be a fatal error). + client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") + ''; + +} diff -Nru nix-2.8.0/tests/nix_path.sh nix-2.16.1+dfsg/tests/nix_path.sh --- nix-2.8.0/tests/nix_path.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nix_path.sh 2023-06-06 10:55:23.000000000 +0000 @@ -9,3 +9,6 @@ # Should ideally also test this, but there’s no pure way to do it, so just trust me that it works # nix-instantiate --eval -E '' -I nixpkgs=channel:nixos-unstable --restrict-eval + +[[ $(nix-instantiate --find-file by-absolute-path/simple.nix) = $PWD/simple.nix ]] +[[ $(nix-instantiate --find-file by-relative-path/simple.nix) = $PWD/simple.nix ]] diff -Nru nix-2.8.0/tests/nix-profile.sh nix-2.16.1+dfsg/tests/nix-profile.sh --- nix-2.8.0/tests/nix-profile.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nix-profile.sh 2023-06-06 10:55:23.000000000 +0000 @@ -3,7 +3,7 @@ clearStore clearProfiles -enableFeatures "ca-derivations ca-references" +enableFeatures "ca-derivations" restartDaemon # Make a flake. @@ -17,6 +17,7 @@ outputs = { self }: with import ./config.nix; rec { packages.$system.default = mkDerivation { name = "profile-test-\${builtins.readFile ./version}"; + outputs = [ "out" "man" "dev" ]; builder = builtins.toFile "builder.sh" '' mkdir -p \$out/bin @@ -26,10 +27,13 @@ EOF chmod +x \$out/bin/hello echo DONE + mkdir -p \$man/share/man + mkdir -p \$dev/include ''; __contentAddressed = import ./ca.nix; outputHashMode = "recursive"; outputHashAlgo = "sha256"; + meta.outputsToInstall = [ "out" "man" ]; }; }; } @@ -46,16 +50,26 @@ nix profile list | grep '0 - - .*-foo-1.0' nix profile install $flake1Dir -L [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +[ -e $TEST_HOME/.nix-profile/share/man ] +(! [ -e $TEST_HOME/.nix-profile/include ]) nix profile history nix profile history | grep "packages.$system.default: ∅ -> 1.0" nix profile diff-closures | grep 'env-manifest.nix: ε → ∅' +# Test XDG Base Directories support + +export NIX_CONFIG="use-xdg-base-directories = true" +nix profile remove 1 +nix profile install $flake1Dir +[[ $($TEST_HOME/.local/state/nix/profile/bin/hello) = "Hello World" ]] +unset NIX_CONFIG + # Test upgrading a package. printf NixOS > $flake1Dir/who printf 2.0 > $flake1Dir/version nix profile upgrade 1 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]] -nix profile history | grep "packages.$system.default: 1.0 -> 2.0" +nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 2.0, 2.0-man" # Test 'history', 'diff-closures'. nix profile diff-closures @@ -86,7 +100,7 @@ printf true > $flake1Dir/ca.nix printf 3.0 > $flake1Dir/version nix profile upgrade 0 -nix profile history | grep "packages.$system.default: 1.0 -> 3.0" +nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man" # Test new install of CA package. nix profile remove 0 @@ -95,3 +109,78 @@ nix profile install $flake1Dir [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]] [[ $(nix path-info --json $(realpath $TEST_HOME/.nix-profile/bin/hello) | jq -r .[].ca) =~ fixed:r:sha256: ]] + +# Override the outputs. +nix profile remove 0 1 +nix profile install "$flake1Dir^*" +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]] +[ -e $TEST_HOME/.nix-profile/share/man ] +[ -e $TEST_HOME/.nix-profile/include ] + +printf Nix > $flake1Dir/who +nix profile upgrade 0 +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Nix" ]] +[ -e $TEST_HOME/.nix-profile/share/man ] +[ -e $TEST_HOME/.nix-profile/include ] + +nix profile remove 0 +nix profile install "$flake1Dir^man" +(! [ -e $TEST_HOME/.nix-profile/bin/hello ]) +[ -e $TEST_HOME/.nix-profile/share/man ] +(! [ -e $TEST_HOME/.nix-profile/include ]) + +# test priority +nix profile remove 0 + +# Make another flake. +flake2Dir=$TEST_ROOT/flake2 +printf World > $flake1Dir/who +cp -r $flake1Dir $flake2Dir +printf World2 > $flake2Dir/who + +nix profile install $flake1Dir +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +expect 1 nix profile install $flake2Dir +diff -u <( + nix --offline profile install $flake2Dir 2>&1 1> /dev/null \ + | grep -vE "^warning: " \ + | grep -vE "^error \(ignored\): " \ + || true +) <(cat << EOF +error: An existing package already provides the following file: + + $(nix build --no-link --print-out-paths ${flake1Dir}"#default.out")/bin/hello + + This is the conflicting file from the new package: + + $(nix build --no-link --print-out-paths ${flake2Dir}"#default.out")/bin/hello + + To remove the existing package: + + nix profile remove path:${flake1Dir}#packages.${system}.default + + The new package can also be installed next to the existing one by assigning a different priority. + The conflicting packages have a priority of 5. + To prioritise the new package: + + nix profile install path:${flake2Dir}#packages.${system}.default --priority 4 + + To prioritise the existing package: + + nix profile install path:${flake2Dir}#packages.${system}.default --priority 6 +EOF +) +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +nix profile install $flake2Dir --priority 100 +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +nix profile install $flake2Dir --priority 0 +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World2" ]] +# nix profile install $flake1Dir --priority 100 +# [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] + +# Ensure that conflicts are handled properly even when the installables aren't +# flake references. +# Regression test for https://github.com/NixOS/nix/issues/8284 +clearProfiles +nix profile install $(nix build $flake1Dir --no-link --print-out-paths) +expect 1 nix profile install --impure --expr "(builtins.getFlake ''$flake2Dir'').packages.$system.default" diff -Nru nix-2.8.0/tests/nix-shell.sh nix-2.16.1+dfsg/tests/nix-shell.sh --- nix-2.8.0/tests/nix-shell.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/nix-shell.sh 2023-06-06 10:55:23.000000000 +0000 @@ -88,17 +88,60 @@ nix develop -f "$shellDotNix" shellDrv -c bash -c '[[ -n $stdenv ]]' # Ensure `nix develop -c` preserves stdin -echo foo | nix develop -f "$shellDotNix" shellDrv -c cat | grep -q foo +echo foo | nix develop -f "$shellDotNix" shellDrv -c cat | grepQuiet foo # Ensure `nix develop -c` actually executes the command if stdout isn't a terminal -nix develop -f "$shellDotNix" shellDrv -c echo foo |& grep -q foo +nix develop -f "$shellDotNix" shellDrv -c echo foo |& grepQuiet foo # Test 'nix print-dev-env'. -[[ $(nix print-dev-env -f "$shellDotNix" shellDrv --json | jq -r .variables.arr1.value[2]) = '3 4' ]] -source <(nix print-dev-env -f "$shellDotNix" shellDrv) -[[ -n $stdenv ]] -[[ ${arr1[2]} = "3 4" ]] -[[ ${arr2[1]} = $'\n' ]] -[[ ${arr2[2]} = $'x\ny' ]] -[[ $(fun) = blabla ]] +nix print-dev-env -f "$shellDotNix" shellDrv > $TEST_ROOT/dev-env.sh +nix print-dev-env -f "$shellDotNix" shellDrv --json > $TEST_ROOT/dev-env.json + +# Test with raw drv + +shellDrv=$(nix-instantiate "$shellDotNix" -A shellDrv.out) + +nix develop $shellDrv -c bash -c '[[ -n $stdenv ]]' + +nix print-dev-env $shellDrv > $TEST_ROOT/dev-env2.sh +nix print-dev-env $shellDrv --json > $TEST_ROOT/dev-env2.json + +diff $TEST_ROOT/dev-env{,2}.sh +diff $TEST_ROOT/dev-env{,2}.json + +# Ensure `nix print-dev-env --json` contains variable assignments. +[[ $(jq -r .variables.arr1.value[2] $TEST_ROOT/dev-env.json) = '3 4' ]] + +# Run tests involving `source <(nix print-dev-inv)` in subshells to avoid modifying the current +# environment. + +set +u # FIXME: Make print-dev-env `set -u` compliant (issue #7951) + +# Ensure `source <(nix print-dev-env)` modifies the environment. +( + path=$PATH + source $TEST_ROOT/dev-env.sh + [[ -n $stdenv ]] + [[ ${arr1[2]} = "3 4" ]] + [[ ${arr2[1]} = $'\n' ]] + [[ ${arr2[2]} = $'x\ny' ]] + [[ $(fun) = blabla ]] + [[ $PATH = $(jq -r .variables.PATH.value $TEST_ROOT/dev-env.json):$path ]] +) + +# Ensure `source <(nix print-dev-env)` handles the case when PATH is empty. +( + path=$PATH + PATH= + source $TEST_ROOT/dev-env.sh + [[ $PATH = $(PATH=$path jq -r .variables.PATH.value $TEST_ROOT/dev-env.json) ]] +) + +# Test nix-shell with ellipsis and no `inNixShell` argument (for backwards compat with old nixpkgs) +cat >$TEST_ROOT/shell-ellipsis.nix < "$out/index.html" - ''; - }; - }; - }; - - # client consumes a remote resolver - client = { lib, nodes, pkgs, ... }: { - networking.useDHCP = false; - networking.nameservers = [ - (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv6.addresses).address - (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv4.addresses).address - ]; - networking.interfaces.eth1.ipv6.addresses = [ - { address = "fd21::10"; prefixLength = 64; } - ]; - networking.interfaces.eth1.ipv4.addresses = [ - { address = "192.168.0.10"; prefixLength = 24; } - ]; - - nix.sandboxPaths = lib.mkForce []; - nix.binaryCaches = lib.mkForce []; - nix.useSandbox = lib.mkForce true; - }; - }; - - nix-fetch = pkgs.writeText "fetch.nix" '' - derivation { - # This derivation is an copy from what is available over at - # nix.git:corepkgs/fetchurl.nix - builder = "builtin:fetchurl"; - - # We're going to fetch data from the http_dns instance created before - # we expect the content to be the same as the content available there. - # ``` - # $ nix-hash --type sha256 --to-base32 $(echo "hello world" | sha256sum | cut -d " " -f 1) - # 0ix4jahrkll5zg01wandq78jw3ab30q4nscph67rniqg5x7r0j59 - # ``` - outputHash = "0ix4jahrkll5zg01wandq78jw3ab30q4nscph67rniqg5x7r0j59"; - outputHashAlgo = "sha256"; - outputHashMode = "flat"; - - name = "example.com"; - url = "http://example.com"; - - unpack = false; - executable = false; - - system = "builtin"; - - preferLocalBuild = true; - - impureEnvVars = [ - "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" - ]; - - urls = [ "http://example.com" ]; - } - ''; - - testScript = { nodes, ... }: '' - http_dns.wait_for_unit("nginx") - http_dns.wait_for_open_port(80) - http_dns.wait_for_unit("unbound") - http_dns.wait_for_open_port(53) - - client.start() - client.wait_for_unit('multi-user.target') - - with subtest("can fetch data from a remote server outside sandbox"): - client.succeed("nix --version >&2") - client.succeed("curl -vvv http://example.com/index.html >&2") - - with subtest("nix-build can lookup dns and fetch data"): - client.succeed(""" - nix-build ${nix-fetch} >&2 - """) - ''; -}) diff -Nru nix-2.8.0/tests/output-normalization.sh nix-2.16.1+dfsg/tests/output-normalization.sh --- nix-2.8.0/tests/output-normalization.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/output-normalization.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,9 @@ +source common.sh + +testNormalization () { + clearStore + outPath=$(nix-build ./simple.nix --no-out-link) + test "$(stat -c %Y $outPath)" -eq 1 +} + +testNormalization diff -Nru nix-2.8.0/tests/path-from-hash-part.sh nix-2.16.1+dfsg/tests/path-from-hash-part.sh --- nix-2.8.0/tests/path-from-hash-part.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/path-from-hash-part.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,10 @@ +source common.sh + +path=$(nix build --no-link --print-out-paths -f simple.nix) + +hash_part=$(basename $path) +hash_part=${hash_part:0:32} + +path2=$(nix store path-from-hash-part $hash_part) + +[[ $path = $path2 ]] diff -Nru nix-2.8.0/tests/plugins/local.mk nix-2.16.1+dfsg/tests/plugins/local.mk --- nix-2.8.0/tests/plugins/local.mk 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/plugins/local.mk 2023-06-06 10:55:23.000000000 +0000 @@ -8,4 +8,4 @@ libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 -libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr +libplugintest_CXXFLAGS := -I src/libutil -I src/libstore -I src/libexpr -I src/libfetchers diff -Nru nix-2.8.0/tests/plugins/plugintest.cc nix-2.16.1+dfsg/tests/plugins/plugintest.cc --- nix-2.8.0/tests/plugins/plugintest.cc 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/plugins/plugintest.cc 2023-06-06 10:55:23.000000000 +0000 @@ -13,7 +13,7 @@ static GlobalConfig::Register rs(&mySettings); -static void prim_anotherNull (EvalState & state, const Pos & pos, Value ** args, Value & v) +static void prim_anotherNull (EvalState & state, const PosIdx pos, Value ** args, Value & v) { if (mySettings.settingSet) v.mkNull(); diff -Nru nix-2.8.0/tests/plugins.sh nix-2.16.1+dfsg/tests/plugins.sh --- nix-2.8.0/tests/plugins.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/plugins.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,8 @@ source common.sh -set -o pipefail +if [[ $BUILD_SHARED_LIBS != 1 ]]; then + skipTest "Plugins are not supported" +fi res=$(nix --option setting-set true --option plugin-files $PWD/plugins/libplugintest* eval --expr builtins.anotherNull) diff -Nru nix-2.8.0/tests/post-hook.sh nix-2.16.1+dfsg/tests/post-hook.sh --- nix-2.8.0/tests/post-hook.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/post-hook.sh 2023-06-06 10:55:23.000000000 +0000 @@ -9,12 +9,23 @@ restartDaemon -# Build the dependencies and push them to the remote store -nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh +if isDaemonNewer "2.13"; then + pushToStore="$PWD/push-to-store.sh" +else + pushToStore="$PWD/push-to-store-old.sh" +fi + +# Build the dependencies and push them to the remote store. +nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook "$pushToStore" +# See if all outputs are passed to the post-build hook by only specifying one +# We're not able to test CA tests this way +export BUILD_HOOK_ONLY_OUT_PATHS=$([ ! $NIX_TESTS_CA_BY_DEFAULT ]) +nix-build -o $TEST_ROOT/result-mult multiple-outputs.nix -A a.first --post-build-hook "$pushToStore" clearStore -# Ensure that we the remote store contains both the runtime and buildtime -# closure of what we've just built +# Ensure that the remote store contains both the runtime and build-time +# closure of what we've just built. nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv +nix copy --from "$REMOTE_STORE" --no-require-sigs -f multiple-outputs.nix a^second diff -Nru nix-2.8.0/tests/pure-eval.sh nix-2.16.1+dfsg/tests/pure-eval.sh --- nix-2.8.0/tests/pure-eval.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/pure-eval.sh 2023-06-06 10:55:23.000000000 +0000 @@ -8,7 +8,7 @@ missingImpureErrorMsg=$(! nix eval --expr 'builtins.readFile ./pure-eval.sh' 2>&1) -echo "$missingImpureErrorMsg" | grep -q -- --impure || \ +echo "$missingImpureErrorMsg" | grepQuiet -- --impure || \ fail "The error message should mention the “--impure” flag to unblock users" [[ $(nix eval --expr 'builtins.pathExists ./pure-eval.sh') == false ]] || \ @@ -30,3 +30,5 @@ rm -rf $TEST_ROOT/eval-out (! nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ "." = "bla"; }') + +(! nix eval --expr '~/foo') diff -Nru nix-2.8.0/tests/push-to-store-old.sh nix-2.16.1+dfsg/tests/push-to-store-old.sh --- nix-2.8.0/tests/push-to-store-old.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/push-to-store-old.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,14 @@ +#!/bin/sh + +set -x +set -e + +[ -n "$OUT_PATHS" ] +[ -n "$DRV_PATH" ] + +echo Pushing "$OUT_PATHS" to "$REMOTE_STORE" +if [ -n "$BUILD_HOOK_ONLY_OUT_PATHS" ]; then + printf "%s" "$OUT_PATHS" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs +else + printf "%s" "$DRV_PATH" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs +fi diff -Nru nix-2.8.0/tests/push-to-store.sh nix-2.16.1+dfsg/tests/push-to-store.sh --- nix-2.8.0/tests/push-to-store.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/push-to-store.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,14 @@ #!/bin/sh set -x +set -e + +[ -n "$OUT_PATHS" ] +[ -n "$DRV_PATH" ] echo Pushing "$OUT_PATHS" to "$REMOTE_STORE" -printf "%s" "$DRV_PATH" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs +if [ -n "$BUILD_HOOK_ONLY_OUT_PATHS" ]; then + printf "%s" "$OUT_PATHS" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs +else + printf "%s" "$DRV_PATH"^'*' | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs +fi diff -Nru nix-2.8.0/tests/readfile-context.builder.sh nix-2.16.1+dfsg/tests/readfile-context.builder.sh --- nix-2.8.0/tests/readfile-context.builder.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/readfile-context.builder.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -echo "$input" > $out diff -Nru nix-2.8.0/tests/readfile-context.nix nix-2.16.1+dfsg/tests/readfile-context.nix --- nix-2.8.0/tests/readfile-context.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/readfile-context.nix 2023-06-06 10:55:23.000000000 +0000 @@ -6,14 +6,23 @@ dependent = mkDerivation { name = "dependent"; - builder = ./readfile-context.builder.sh; - input = "${input}/hello"; + buildCommand = '' + mkdir -p $out + echo -n "$input1" > "$out/file1" + echo -n "$input2" > "$out/file2" + ''; + input1 = "${input}/hello"; + input2 = "hello"; }; readDependent = mkDerivation { - name = "read-dependent"; - builder = ./readfile-context.builder.sh; - input = builtins.readFile dependent; + # Will evaluate correctly because file2 doesn't have any references, + # even though the `dependent` derivation does. + name = builtins.readFile (dependent + "/file2"); + buildCommand = '' + echo "$input" > "$out" + ''; + input = builtins.readFile (dependent + "/file1"); }; in readDependent diff -Nru nix-2.8.0/tests/recursive.nix nix-2.16.1+dfsg/tests/recursive.nix --- nix-2.8.0/tests/recursive.nix 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/recursive.nix 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,56 @@ +with import ./config.nix; + +mkDerivation rec { + name = "recursive"; + dummy = builtins.toFile "dummy" "bla bla"; + SHELL = shell; + + # Note: this is a string without context. + unreachable = builtins.getEnv "unreachable"; + + NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT"; + + requiredSystemFeatures = [ "recursive-nix" ]; + + buildCommand = '' + mkdir $out + opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}" + + PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + + # Check that we can query/build paths in our input closure. + nix $opts path-info $dummy + nix $opts build $dummy + + # Make sure we cannot query/build paths not in out input closure. + [[ -e $unreachable ]] + (! nix $opts path-info $unreachable) + (! nix $opts build $unreachable) + + # Add something to the store. + echo foobar > foobar + foobar=$(nix $opts store add-path ./foobar) + + nix $opts path-info $foobar + nix $opts build $foobar + + # Add it to our closure. + ln -s $foobar $out/foobar + + [[ $(nix $opts path-info --all | wc -l) -eq 4 ]] + + # Build a derivation. + nix $opts build -L --impure --expr ' + with import ${./config.nix}; + mkDerivation { + name = "inner1"; + buildCommand = "echo $fnord blaat > $out"; + fnord = builtins.toFile "fnord" "fnord"; + } + ' + + [[ $(nix $opts path-info --json ./result) =~ fnord ]] + + ln -s $(nix $opts path-info ./result) $out/inner1 + ''; +} diff -Nru nix-2.8.0/tests/recursive.sh nix-2.16.1+dfsg/tests/recursive.sh --- nix-2.8.0/tests/recursive.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/recursive.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,10 +1,10 @@ source common.sh -sed -i 's/experimental-features .*/& recursive-nix/' "$NIX_CONF_DIR"/nix.conf -restartDaemon - # FIXME -if [[ $(uname) != Linux ]]; then exit 99; fi +if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi + +enableFeatures 'recursive-nix' +restartDaemon clearStore @@ -12,63 +12,7 @@ export unreachable=$(nix store add-path ./recursive.sh) -NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' - with import ./config.nix; - mkDerivation rec { - name = "recursive"; - dummy = builtins.toFile "dummy" "bla bla"; - SHELL = shell; - - # Note: this is a string without context. - unreachable = builtins.getEnv "unreachable"; - - NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT"; - - requiredSystemFeatures = [ "recursive-nix" ]; - - buildCommand = '\'\'' - mkdir $out - opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}" - - PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH - - # Check that we can query/build paths in our input closure. - nix $opts path-info $dummy - nix $opts build $dummy - - # Make sure we cannot query/build paths not in out input closure. - [[ -e $unreachable ]] - (! nix $opts path-info $unreachable) - (! nix $opts build $unreachable) - - # Add something to the store. - echo foobar > foobar - foobar=$(nix $opts store add-path ./foobar) - - nix $opts path-info $foobar - nix $opts build $foobar - - # Add it to our closure. - ln -s $foobar $out/foobar - - [[ $(nix $opts path-info --all | wc -l) -eq 4 ]] - - # Build a derivation. - nix $opts build -L --impure --expr '\'' - with import ${./config.nix}; - mkDerivation { - name = "inner1"; - buildCommand = "echo $fnord blaat > $out"; - fnord = builtins.toFile "fnord" "fnord"; - } - '\'' - - [[ $(nix $opts path-info --json ./result) =~ fnord ]] - - ln -s $(nix $opts path-info ./result) $out/inner1 - '\'\''; - } -' +NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --file ./recursive.nix [[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]] diff -Nru nix-2.8.0/tests/remote-builds.nix nix-2.16.1+dfsg/tests/remote-builds.nix --- nix-2.8.0/tests/remote-builds.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/remote-builds.nix 1970-01-01 00:00:00.000000000 +0000 @@ -1,112 +0,0 @@ -# Test Nix's remote build feature. - -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - -makeTest ( - -let - - # The configuration of the remote builders. - builder = - { config, pkgs, ... }: - { services.openssh.enable = true; - virtualisation.writableStore = true; - nix.useSandbox = true; - }; - - # Trivial Nix expression to build remotely. - expr = config: nr: pkgs.writeText "expr.nix" - '' - let utils = builtins.storePath ${config.system.build.extraUtils}; in - derivation { - name = "hello-${toString nr}"; - system = "i686-linux"; - PATH = "''${utils}/bin"; - builder = "''${utils}/bin/sh"; - args = [ "-c" "if [ ${toString nr} = 5 ]; then echo FAIL; exit 1; fi; echo Hello; mkdir $out $foo; cat /proc/sys/kernel/hostname > $out/host; ln -s $out $foo/bar; sleep 10" ]; - outputs = [ "out" "foo" ]; - } - ''; - -in - -{ - name = "remote-builds"; - - nodes = - { builder1 = builder; - builder2 = builder; - - client = - { config, lib, pkgs, ... }: - { nix.maxJobs = 0; # force remote building - nix.distributedBuilds = true; - nix.buildMachines = - [ { hostName = "builder1"; - sshUser = "root"; - sshKey = "/root/.ssh/id_ed25519"; - system = "i686-linux"; - maxJobs = 1; - } - { hostName = "builder2"; - sshUser = "root"; - sshKey = "/root/.ssh/id_ed25519"; - system = "i686-linux"; - maxJobs = 1; - } - ]; - virtualisation.writableStore = true; - virtualisation.pathsInNixDB = [ config.system.build.extraUtils ]; - nix.binaryCaches = lib.mkForce [ ]; - programs.ssh.extraConfig = "ConnectTimeout 30"; - }; - }; - - testScript = { nodes }: '' - # fmt: off - import subprocess - - start_all() - - # Create an SSH key on the client. - subprocess.run([ - "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" - ], capture_output=True, check=True) - client.succeed("mkdir -p -m 700 /root/.ssh") - client.copy_from_host("key", "/root/.ssh/id_ed25519") - client.succeed("chmod 600 /root/.ssh/id_ed25519") - - # Install the SSH key on the builders. - client.wait_for_unit("network.target") - for builder in [builder1, builder2]: - builder.succeed("mkdir -p -m 700 /root/.ssh") - builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") - builder.wait_for_unit("sshd") - client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") - - # Perform a build and check that it was performed on the builder. - out = client.succeed( - "nix-build ${expr nodes.client.config 1} 2> build-output", - "grep -q Hello build-output" - ) - builder1.succeed(f"test -e {out}") - - # And a parallel build. - paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out') - out1, out2 = paths.split() - builder1.succeed(f"test -e {out1} -o -e {out2}") - builder2.succeed(f"test -e {out1} -o -e {out2}") - - # And a failing build. - client.fail("nix-build ${expr nodes.client.config 5}") - - # Test whether the build hook automatically skips unavailable builders. - builder1.block() - client.succeed("nix-build ${expr nodes.client.config 4}") - ''; -}) diff -Nru nix-2.8.0/tests/remote-store.sh nix-2.16.1+dfsg/tests/remote-store.sh --- nix-2.8.0/tests/remote-store.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/remote-store.sh 2023-06-06 10:55:23.000000000 +0000 @@ -5,8 +5,19 @@ # Ensure "fake ssh" remote store works just as legacy fake ssh would. nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor +# Ensure that store ping trusted works with ssh-ng:// +nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e '.trusted' + startDaemon +if isDaemonNewer "2.15pre0"; then + # Ensure that ping works trusted with new daemon + nix store ping --json | jq -e '.trusted' +else + # And the the field is absent with the old daemon + nix store ping --json | jq -e 'has("trusted") | not' +fi + # Test import-from-derivation through the daemon. [[ $(nix eval --impure --raw --expr ' with import ./config.nix; @@ -30,7 +41,3 @@ cmp $TEST_ROOT/d1 $TEST_ROOT/d2 killDaemon - -user=$(whoami) -[ -e $NIX_STATE_DIR/gcroots/per-user/$user ] -[ -e $NIX_STATE_DIR/profiles/per-user/$user ] diff -Nru nix-2.8.0/tests/repl.sh nix-2.16.1+dfsg/tests/repl.sh --- nix-2.8.0/tests/repl.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/repl.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,39 +1,52 @@ source common.sh +testDir="$PWD" +cd "$TEST_ROOT" + replCmds=" simple = 1 -simple = import ./simple.nix -:b simple +simple = import $testDir/simple.nix +:bl simple :log simple " replFailingCmds=" -failing = import ./simple-failing.nix +failing = import $testDir/simple-failing.nix :b failing :log failing " replUndefinedVariable=" -import ./undefined-variable.nix +import $testDir/undefined-variable.nix " testRepl () { local nixArgs=("$@") + rm -rf repl-result-out || true # cleanup from other runs backed by a foreign nix store local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replCmds")" echo "$replOutput" local outPath=$(echo "$replOutput" |& grep -o -E "$NIX_STORE_DIR/\w*-simple") nix path-info "${nixArgs[@]}" "$outPath" + [ "$(realpath ./repl-result-out)" == "$outPath" ] || fail "nix repl :bl doesn't make a symlink" + # run it again without checking the output to ensure the previously created symlink gets overwritten + nix repl "${nixArgs[@]}" <<< "$replCmds" || fail "nix repl does not work twice with the same inputs" + # simple.nix prints a PATH during build - echo "$replOutput" | grep -qs 'PATH=' || fail "nix repl :log doesn't output logs" + echo "$replOutput" | grepQuiet -s 'PATH=' || fail "nix repl :log doesn't output logs" local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replFailingCmds" 2>&1)" echo "$replOutput" - echo "$replOutput" | grep -qs 'This should fail' \ + echo "$replOutput" | grepQuiet -s 'This should fail' \ || fail "nix repl :log doesn't output logs for a failed derivation" local replOutput="$(nix repl --show-trace "${nixArgs[@]}" <<< "$replUndefinedVariable" 2>&1)" echo "$replOutput" - echo "$replOutput" | grep -qs "while evaluating the file" \ + echo "$replOutput" | grepQuiet -s "while evaluating the file" \ || fail "nix repl --show-trace doesn't show the trace" + + nix repl "${nixArgs[@]}" --option pure-eval true 2>&1 <<< "builtins.currentSystem" \ + | grep "attribute 'currentSystem' missing" + nix repl "${nixArgs[@]}" 2>&1 <<< "builtins.currentSystem" \ + | grep "$(nix-instantiate --eval -E 'builtins.currentSystem')" } # Simple test, try building a drv @@ -42,15 +55,17 @@ testRepl --store "$TEST_ROOT/store?real=$NIX_STORE_DIR" testReplResponse () { - local response="$(nix repl <<< "$1")" - echo "$response" | grep -qs "$2" \ + local commands="$1"; shift + local expectedResponse="$1"; shift + local response="$(nix repl "$@" <<< "$commands")" + echo "$response" | grepQuiet -s "$expectedResponse" \ || fail "repl command set: -$1 +$commands does not respond with: -$2 +$expectedResponse but with: @@ -63,3 +78,56 @@ :a { a = "2"; } "result: ${a}" ' "result: 2" + +# check dollar escaping https://github.com/NixOS/nix/issues/4909 +# note the escaped \, +# \\ +# because the second argument is a regex +testReplResponse ' +"$" + "{hi}" +' '"\\${hi}"' + +testReplResponse ' +drvPath +' '".*-simple.drv"' \ +$testDir/simple.nix + +testReplResponse ' +drvPath +' '".*-simple.drv"' \ +--file $testDir/simple.nix --experimental-features 'ca-derivations' + +testReplResponse ' +drvPath +' '".*-simple.drv"' \ +--file $testDir/simple.nix --extra-experimental-features 'repl-flake ca-derivations' + +mkdir -p flake && cat < flake/flake.nix +{ + outputs = { self }: { + foo = 1; + bar.baz = 2; + + changingThing = "beforeChange"; + }; +} +EOF +testReplResponse ' +foo + baz +' "3" \ + ./flake ./flake\#bar --experimental-features 'flakes repl-flake' + +# Test the `:reload` mechansim with flakes: +# - Eval `./flake#changingThing` +# - Modify the flake +# - Re-eval it +# - Check that the result has changed +replResult=$( ( +echo "changingThing" +sleep 1 # Leave the repl the time to eval 'foo' +sed -i 's/beforeChange/afterChange/' flake/flake.nix +echo ":reload" +echo "changingThing" +) | nix repl ./flake --experimental-features 'flakes repl-flake') +echo "$replResult" | grepQuiet -s beforeChange +echo "$replResult" | grepQuiet -s afterChange diff -Nru nix-2.8.0/tests/restricted.sh nix-2.16.1+dfsg/tests/restricted.sh --- nix-2.8.0/tests/restricted.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/restricted.sh 2023-06-06 10:55:23.000000000 +0000 @@ -3,7 +3,7 @@ clearStore nix-instantiate --restrict-eval --eval -E '1 + 2' -(! nix-instantiate --restrict-eval ./restricted.nix) +(! nix-instantiate --eval --restrict-eval ./restricted.nix) (! nix-instantiate --eval --restrict-eval <(echo '1 + 2')) nix-instantiate --restrict-eval ./simple.nix -I src=. nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh @@ -48,4 +48,4 @@ --expr "builtins.readFile \"$traverseDir/$goUp$(pwd)/restricted-innocent\"" \ 2>&1 || :)" echo "$output" | grep "is forbidden" -! echo "$output" | grep -F restricted-secret +echo "$output" | grepInverse -F restricted-secret diff -Nru nix-2.8.0/tests/search.sh nix-2.16.1+dfsg/tests/search.sh --- nix-2.8.0/tests/search.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/search.sh 2023-06-06 10:55:23.000000000 +0000 @@ -20,19 +20,27 @@ ## Search expressions # Check that empty search string matches all -nix search -f search.nix '' |grep -q foo -nix search -f search.nix '' |grep -q bar -nix search -f search.nix '' |grep -q hello +nix search -f search.nix '' |grepQuiet foo +nix search -f search.nix '' |grepQuiet bar +nix search -f search.nix '' |grepQuiet hello ## Tests for multiple regex/match highlighting e=$'\x1b' # grep doesn't support \e, \033 or even \x1b # Multiple overlapping regexes -(( $(nix search -f search.nix '' 'oo' 'foo' 'oo' | grep "$e\[32;1mfoo$e\\[0;1m" | wc -l) == 1 )) -(( $(nix search -f search.nix '' 'broken b' 'en bar' | grep "$e\[32;1mbroken bar$e\\[0m" | wc -l) == 1 )) +(( $(nix search -f search.nix '' 'oo' 'foo' 'oo' | grep -c "$e\[32;1mfoo$e\\[0;1m") == 1 )) +(( $(nix search -f search.nix '' 'broken b' 'en bar' | grep -c "$e\[32;1mbroken bar$e\\[0m") == 1 )) # Multiple matches # Searching for 'o' should yield the 'o' in 'broken bar', the 'oo' in foo and 'o' in hello -(( $(nix search -f search.nix '' 'o' | grep -Eo "$e\[32;1mo{1,2}$e\[(0|0;1)m" | wc -l) == 3 )) +(( $(nix search -f search.nix '' 'o' | grep -Eoc "$e\[32;1mo{1,2}$e\[(0|0;1)m") == 3 )) # Searching for 'b' should yield the 'b' in bar and the two 'b's in 'broken bar' +# NOTE: This does not work with `grep -c` because it counts the two 'b's in 'broken bar' as one matched line (( $(nix search -f search.nix '' 'b' | grep -Eo "$e\[32;1mb$e\[(0|0;1)m" | wc -l) == 3 )) + +## Tests for --exclude +(( $(nix search -f search.nix -e hello | grep -c hello) == 0 )) + +(( $(nix search -f search.nix foo --exclude 'foo|bar' | grep -Ec 'foo|bar') == 0 )) +(( $(nix search -f search.nix foo -e foo --exclude bar | grep -Ec 'foo|bar') == 0 )) +[[ $(nix search -f search.nix -e bar --json | jq -c 'keys') == '["foo","hello"]' ]] diff -Nru nix-2.8.0/tests/selfref-gc.sh nix-2.16.1+dfsg/tests/selfref-gc.sh --- nix-2.8.0/tests/selfref-gc.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/selfref-gc.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,30 @@ +source common.sh + +requireDaemonNewerThan "2.6.0pre20211215" + +clearStore + +nix-build --no-out-link -E ' + with import ./config.nix; + + let d1 = mkDerivation { + name = "selfref-gc"; + outputs = [ "out" ]; + buildCommand = " + echo SELF_REF: $out > $out + "; + }; in + + # the only change from d1 is d1 as an (unused) build input + # to get identical store path in CA. + mkDerivation { + name = "selfref-gc"; + outputs = [ "out" ]; + buildCommand = " + echo UNUSED: ${d1} + echo SELF_REF: $out > $out + "; + } +' + +nix-collect-garbage diff -Nru nix-2.8.0/tests/setuid.nix nix-2.16.1+dfsg/tests/setuid.nix --- nix-2.8.0/tests/setuid.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/setuid.nix 1970-01-01 00:00:00.000000000 +0000 @@ -1,126 +0,0 @@ -# Verify that Linux builds cannot create setuid or setgid binaries. - -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - -makeTest { - name = "setuid"; - - machine = - { config, lib, pkgs, ... }: - { virtualisation.writableStore = true; - nix.binaryCaches = lib.mkForce [ ]; - nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ]; - virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; - }; - - testScript = { nodes }: '' - # fmt: off - start_all() - - # Copying to /tmp should succeed. - machine.succeed(r""" - nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - ")' - """.strip()) - - machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - - machine.succeed("rm /tmp/id") - - # Creating a setuid binary should fail. - machine.fail(r""" - nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - chmod 4755 /tmp/id - ")' - """.strip()) - - machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - - machine.succeed("rm /tmp/id") - - # Creating a setgid binary should fail. - machine.fail(r""" - nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - chmod 2755 /tmp/id - ")' - """.strip()) - - machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - - machine.succeed("rm /tmp/id") - - # The checks should also work on 32-bit binaries. - machine.fail(r""" - nix-build --no-sandbox -E '(with import { system = "i686-linux"; }; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - chmod 2755 /tmp/id - ")' - """.strip()) - - machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - - machine.succeed("rm /tmp/id") - - # The tests above use fchmodat(). Test chmod() as well. - machine.succeed(r""" - nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"chmod 0666, qw(/tmp/id) or die\" - ")' - """.strip()) - - machine.succeed('[[ $(stat -c %a /tmp/id) = 666 ]]') - - machine.succeed("rm /tmp/id") - - machine.fail(r""" - nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"chmod 04755, qw(/tmp/id) or die\" - ")' - """.strip()) - - machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - - machine.succeed("rm /tmp/id") - - # And test fchmod(). - machine.succeed(r""" - nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\" - ")' - """.strip()) - - machine.succeed('[[ $(stat -c %a /tmp/id) = 1750 ]]') - - machine.succeed("rm /tmp/id") - - machine.fail(r""" - nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\" - ")' - """.strip()) - - machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - - machine.succeed("rm /tmp/id") - ''; -} diff -Nru nix-2.8.0/tests/shell-hello.nix nix-2.16.1+dfsg/tests/shell-hello.nix --- nix-2.8.0/tests/shell-hello.nix 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/shell-hello.nix 2023-06-06 10:55:23.000000000 +0000 @@ -3,15 +3,24 @@ { hello = mkDerivation { name = "hello"; + outputs = [ "out" "dev" ]; + meta.outputsToInstall = [ "out" ]; buildCommand = '' - mkdir -p $out/bin + mkdir -p $out/bin $dev/bin + cat > $out/bin/hello < $dev/bin/hello2 < $out/HEAD - - mkdir -p $out/info - echo -e '${nixpkgs.rev}\trefs/heads/master' > $out/info/refs - ''; - -in - -makeTest ( - - { - name = "sourcehut-flakes"; - - nodes = - { - # Impersonate git.sr.ht - sourcehut = - { config, pkgs, ... }: - { - networking.firewall.allowedTCPPorts = [ 80 443 ]; - - services.httpd.enable = true; - services.httpd.adminAddr = "foo@example.org"; - services.httpd.extraConfig = '' - ErrorLog syslog:local6 - ''; - services.httpd.virtualHosts."git.sr.ht" = - { - forceSSL = true; - sslServerKey = "${cert}/server.key"; - sslServerCert = "${cert}/server.crt"; - servedDirs = - [ - { - urlPath = "/~NixOS/nixpkgs"; - dir = nixpkgs-repo; - } - { - urlPath = "/~NixOS/flake-registry/blob/master"; - dir = registry; - } - ]; - }; - }; - - client = - { config, lib, pkgs, nodes, ... }: - { - virtualisation.writableStore = true; - virtualisation.diskSize = 2048; - virtualisation.pathsInNixDB = [ pkgs.hello pkgs.fuse ]; - virtualisation.memorySize = 4096; - nix.binaryCaches = lib.mkForce [ ]; - nix.extraOptions = '' - experimental-features = nix-command flakes - flake-registry = https://git.sr.ht/~NixOS/flake-registry/blob/master/flake-registry.json - ''; - environment.systemPackages = [ pkgs.jq ]; - networking.hosts.${(builtins.head nodes.sourcehut.config.networking.interfaces.eth1.ipv4.addresses).address} = - [ "git.sr.ht" ]; - security.pki.certificateFiles = [ "${cert}/ca.crt" ]; - }; - }; - - testScript = { nodes }: '' - # fmt: off - import json - import time - - start_all() - - sourcehut.wait_for_unit("httpd.service") - - client.succeed("curl -v https://git.sr.ht/ >&2") - client.succeed("nix registry list | grep nixpkgs") - - rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision") - assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" - - client.succeed("nix registry pin nixpkgs") - - client.succeed("nix flake info nixpkgs --tarball-ttl 0 >&2") - - # Shut down the web server. The flake should be cached on the client. - sourcehut.succeed("systemctl stop httpd.service") - - info = json.loads(client.succeed("nix flake info nixpkgs --json")) - date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified'])) - assert date == "${nixpkgs.lastModifiedDate}", "time mismatch" - - client.succeed("nix build nixpkgs#hello") - - # The build shouldn't fail even with --tarball-ttl 0 (the server - # being down should not be a fatal error). - client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") - ''; - - }) diff -Nru nix-2.8.0/tests/store-ping.sh nix-2.16.1+dfsg/tests/store-ping.sh --- nix-2.8.0/tests/store-ping.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/store-ping.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,13 +1,17 @@ source common.sh STORE_INFO=$(nix store ping 2>&1) +STORE_INFO_JSON=$(nix store ping --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then DAEMON_VERSION=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3) echo "$STORE_INFO" | grep "Version: $DAEMON_VERSION" + [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi expect 127 NIX_REMOTE=unix:$PWD/store nix store ping || \ fail "nix store ping on a non-existent store should fail" + +[[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]] diff -Nru nix-2.8.0/tests/tarball.sh nix-2.16.1+dfsg/tests/tarball.sh --- nix-2.8.0/tests/tarball.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/tarball.sh 2023-06-06 10:55:23.000000000 +0000 @@ -19,7 +19,7 @@ tarball=$TEST_ROOT/tarball.tar$ext (cd $TEST_ROOT && tar cf - tarball) | $compressor > $tarball - nix-env -f file://$tarball -qa --out-path | grep -q dependencies + nix-env -f file://$tarball -qa --out-path | grepQuiet dependencies nix-build -o $TEST_ROOT/result file://$tarball @@ -34,7 +34,7 @@ nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" # Do not re-fetch paths already present nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input' + expectStderr 102 nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input' nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' diff -Nru nix-2.8.0/tests/test-infra.sh nix-2.16.1+dfsg/tests/test-infra.sh --- nix-2.8.0/tests/test-infra.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/test-infra.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,85 @@ +# Test the functions for testing themselves! +# Also test some assumptions on how bash works that they rely on. +source common.sh + +# `true` should exit with 0 +expect 0 true + +# `false` should exit with 1 +expect 1 false + +# `expect` will fail when we get it wrong +expect 1 expect 0 false + +noisyTrue () { + echo YAY! >&2 + true +} + +noisyFalse () { + echo NAY! >&2 + false +} + +# These should redirect standard error to standard output +expectStderr 0 noisyTrue | grepQuiet YAY +expectStderr 1 noisyFalse | grepQuiet NAY + +# `set -o pipefile` is enabled + +pipefailure () { + # shellcheck disable=SC2216 + true | false | true +} +expect 1 pipefailure +unset pipefailure + +pipefailure () { + # shellcheck disable=SC2216 + false | true | true +} +expect 1 pipefailure +unset pipefailure + +commandSubstitutionPipeFailure () { + # shellcheck disable=SC2216 + res=$(set -eu -o pipefail; false | true | echo 0) +} +expect 1 commandSubstitutionPipeFailure + +# `set -u` is enabled + +# note (...), making function use subshell, as unbound variable errors +# in the outer shell are *rightly* not recoverable. +useUnbound () ( + set -eu + # shellcheck disable=SC2154 + echo "$thisVariableIsNotBound" +) +expect 1 useUnbound + +# ! alone unfortunately negates `set -e`, but it works in functions: +# shellcheck disable=SC2251 +! true +funBang () { + ! true +} +expect 1 funBang +unset funBang + +# `grep -v -q` is not what we want for exit codes, but `grepInverse` is +# Avoid `grep -v -q`. The following line proves the point, and if it fails, +# we'll know that `grep` had a breaking change or `-v -q` may not be portable. +{ echo foo; echo bar; } | grep -v -q foo +{ echo foo; echo bar; } | expect 1 grepInverse foo + +# `grepQuiet` is quiet +res=$(set -eu -o pipefail; echo foo | grepQuiet foo | wc -c) +(( res == 0 )) +unset res + +# `greqQietInverse` is both +{ echo foo; echo bar; } | expect 1 grepQuietInverse foo +res=$(set -eu -o pipefail; echo foo | expect 1 grepQuietInverse foo | wc -c) +(( res == 0 )) +unset res diff -Nru nix-2.8.0/tests/timeout.sh nix-2.16.1+dfsg/tests/timeout.sh --- nix-2.8.0/tests/timeout.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/timeout.sh 2023-06-06 10:55:23.000000000 +0000 @@ -5,17 +5,14 @@ # XXX: This shouldn’t be, but #4813 cause this test to fail needLocalStore "see #4813" -set +e -messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) -status=$? -set -e +messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) && status=0 || status=$? if [ $status -ne 101 ]; then echo "error: 'nix-store' exited with '$status'; should have exited 101" exit 1 fi -if ! echo "$messages" | grep -q "timed out"; then +if echo "$messages" | grepQuietInvert "timed out"; then echo "error: build may have failed for reasons other than timeout; output:" echo "$messages" >&2 exit 1 diff -Nru nix-2.8.0/tests/toString-path.sh nix-2.16.1+dfsg/tests/toString-path.sh --- nix-2.8.0/tests/toString-path.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/toString-path.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,8 @@ +source common.sh + +mkdir -p $TEST_ROOT/foo +echo bla > $TEST_ROOT/foo/bar + +[[ $(nix eval --raw --impure --expr "builtins.readFile (builtins.toString (builtins.fetchTree { type = \"path\"; path = \"$TEST_ROOT/foo\"; } + \"/bar\"))") = bla ]] + +[[ $(nix eval --json --impure --expr "builtins.readDir (builtins.toString (builtins.fetchTree { type = \"path\"; path = \"$TEST_ROOT/foo\"; }))") = '{"bar":"regular"}' ]] diff -Nru nix-2.8.0/tests/user-envs-migration.sh nix-2.16.1+dfsg/tests/user-envs-migration.sh --- nix-2.8.0/tests/user-envs-migration.sh 1970-01-01 00:00:00.000000000 +0000 +++ nix-2.16.1+dfsg/tests/user-envs-migration.sh 2023-06-06 10:55:23.000000000 +0000 @@ -0,0 +1,35 @@ +# Test that the migration of user environments +# (https://github.com/NixOS/nix/pull/5226) does preserve everything + +source common.sh + +if isDaemonNewer "2.4pre20211005"; then + skipTest "Daemon is too new" +fi + + +killDaemon +unset NIX_REMOTE + +clearStore +clearProfiles +rm -rf ~/.nix-profile + +# Fill the environment using the older Nix +PATH_WITH_NEW_NIX="$PATH" +export PATH="$NIX_DAEMON_PACKAGE/bin:$PATH" + +nix-env -f user-envs.nix -i foo-1.0 +nix-env -f user-envs.nix -i bar-0.1 + +# Migrate to the new profile dir, and ensure that everything’s there +export PATH="$PATH_WITH_NEW_NIX" +nix-env -q # Trigger the migration +( [[ -L ~/.nix-profile ]] && \ + [[ $(readlink ~/.nix-profile) == ~/.local/share/nix/profiles/profile ]] ) || \ + fail "The nix profile should point to the new location" + +(nix-env -q | grep foo && nix-env -q | grep bar && \ + [[ -e ~/.nix-profile/bin/foo ]] && \ + [[ $(nix-env --list-generations | wc -l) == 2 ]]) || + fail "The nix profile should have the same content as before the migration" diff -Nru nix-2.8.0/tests/user-envs.sh nix-2.16.1+dfsg/tests/user-envs.sh --- nix-2.8.0/tests/user-envs.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/user-envs.sh 2023-06-06 10:55:23.000000000 +0000 @@ -1,6 +1,6 @@ source common.sh -if [ -z "$storeCleared" ]; then +if [ -z "${storeCleared-}" ]; then clearStore fi @@ -28,13 +28,13 @@ ] | all' # Query descriptions. -nix-env -f ./user-envs.nix -qa '*' --description | grep -q silly +nix-env -f ./user-envs.nix -qa '*' --description | grepQuiet silly rm -rf $HOME/.nix-defexpr ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr -nix-env -qa '*' --description | grep -q silly +nix-env -qa '*' --description | grepQuiet silly # Query the system. -nix-env -qa '*' --system | grep -q $system +nix-env -qa '*' --system | grepQuiet $system # Install "foo-1.0". nix-env -i foo-1.0 @@ -42,19 +42,19 @@ # Query installed: should contain foo-1.0 now (which should be # executable). test "$(nix-env -q '*' | wc -l)" -eq 1 -nix-env -q '*' | grep -q foo-1.0 +nix-env -q '*' | grepQuiet foo-1.0 test "$($profiles/test/bin/foo)" = "foo-1.0" # Test nix-env -qc to compare installed against available packages, and vice versa. -nix-env -qc '*' | grep -q '< 2.0' -nix-env -qac '*' | grep -q '> 1.0' +nix-env -qc '*' | grepQuiet '< 2.0' +nix-env -qac '*' | grepQuiet '> 1.0' # Test the -b flag to filter out source-only packages. [ "$(nix-env -qab | wc -l)" -eq 1 ] # Test the -s flag to get package status. -nix-env -qas | grep -q 'IP- foo-1.0' -nix-env -qas | grep -q -- '--- bar-0.1' +nix-env -qas | grepQuiet 'IP- foo-1.0' +nix-env -qas | grepQuiet -- '--- bar-0.1' # Disable foo. nix-env --set-flag active false foo @@ -74,15 +74,15 @@ # Query installed: should contain foo-2.0pre1 now. test "$(nix-env -q '*' | wc -l)" -eq 1 -nix-env -q '*' | grep -q foo-2.0pre1 +nix-env -q '*' | grepQuiet foo-2.0pre1 test "$($profiles/test/bin/foo)" = "foo-2.0pre1" # Upgrade "foo": should install foo-2.0. -NIX_PATH=nixpkgs=./user-envs.nix:$NIX_PATH nix-env -f '' -u foo +NIX_PATH=nixpkgs=./user-envs.nix:${NIX_PATH-} nix-env -f '' -u foo # Query installed: should contain foo-2.0 now. test "$(nix-env -q '*' | wc -l)" -eq 1 -nix-env -q '*' | grep -q foo-2.0 +nix-env -q '*' | grepQuiet foo-2.0 test "$($profiles/test/bin/foo)" = "foo-2.0" # Store the path of foo-2.0. @@ -94,20 +94,20 @@ nix-env -e foo # Query installed: should only contain bar-0.1 now. -if nix-env -q '*' | grep -q foo; then false; fi -nix-env -q '*' | grep -q bar +if nix-env -q '*' | grepQuiet foo; then false; fi +nix-env -q '*' | grepQuiet bar # Rollback: should bring "foo" back. oldGen="$(nix-store -q --resolve $profiles/test)" nix-env --rollback [ "$(nix-store -q --resolve $profiles/test)" != "$oldGen" ] -nix-env -q '*' | grep -q foo-2.0 -nix-env -q '*' | grep -q bar +nix-env -q '*' | grepQuiet foo-2.0 +nix-env -q '*' | grepQuiet bar # Rollback again: should remove "bar". nix-env --rollback -nix-env -q '*' | grep -q foo-2.0 -if nix-env -q '*' | grep -q bar; then false; fi +nix-env -q '*' | grepQuiet foo-2.0 +if nix-env -q '*' | grepQuiet bar; then false; fi # Count generations. nix-env --list-generations @@ -129,7 +129,7 @@ # Install foo-1.0, now using its store path. nix-env -i "$outPath10" -nix-env -q '*' | grep -q foo-1.0 +nix-env -q '*' | grepQuiet foo-1.0 nix-store -qR $profiles/test | grep "$outPath10" nix-store -q --referrers-closure $profiles/test | grep "$(nix-store -q --resolve $profiles/test)" [ "$(nix-store -q --deriver "$outPath10")" = $drvPath10 ] @@ -137,12 +137,12 @@ # Uninstall foo-1.0, using a symlink to its store path. ln -sfn $outPath10/bin/foo $TEST_ROOT/symlink nix-env -e $TEST_ROOT/symlink -if nix-env -q '*' | grep -q foo; then false; fi -(! nix-store -qR $profiles/test | grep "$outPath10") +if nix-env -q '*' | grepQuiet foo; then false; fi +nix-store -qR $profiles/test | grepInverse "$outPath10" # Install foo-1.0, now using a symlink to its store path. nix-env -i $TEST_ROOT/symlink -nix-env -q '*' | grep -q foo +nix-env -q '*' | grepQuiet foo # Delete all old generations. nix-env --delete-generations old @@ -160,7 +160,7 @@ # Installing "foo" should only install the newest foo. nix-env -i foo test "$(nix-env -q '*' | grep foo- | wc -l)" -eq 1 -nix-env -q '*' | grep -q foo-2.0 +nix-env -q '*' | grepQuiet foo-2.0 # On the other hand, this should install both (and should fail due to # a collision). @@ -171,8 +171,8 @@ nix-env -e '*' nix-env -i '*' test "$(nix-env -q '*' | wc -l)" -eq 2 -nix-env -q '*' | grep -q foo-2.0 -nix-env -q '*' | grep -q bar-0.1.1 +nix-env -q '*' | grepQuiet foo-2.0 +nix-env -q '*' | grepQuiet bar-0.1.1 # Test priorities: foo-0.1 has a lower priority than foo-1.0, so it # should be possible to install both without a collision. Also test diff -Nru nix-2.8.0/tests/why-depends.sh nix-2.16.1+dfsg/tests/why-depends.sh --- nix-2.8.0/tests/why-depends.sh 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/tests/why-depends.sh 2023-06-06 10:55:23.000000000 +0000 @@ -6,6 +6,9 @@ cd $TEST_HOME +nix why-depends --derivation --file ./dependencies.nix input2_drv input1_drv +nix why-depends --file ./dependencies.nix input2_drv input1_drv + nix-build ./dependencies.nix -A input0_drv -o dep nix-build ./dependencies.nix -o toplevel @@ -13,9 +16,9 @@ PRECISE_WHY_DEPENDS_OUTPUT=$(nix why-depends ./toplevel ./dep --precise) # Both outputs should show that `input-2` is in the dependency chain -echo "$FAST_WHY_DEPENDS_OUTPUT" | grep -q input-2 -echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grep -q input-2 +echo "$FAST_WHY_DEPENDS_OUTPUT" | grepQuiet input-2 +echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grepQuiet input-2 -# But only the “precise” one should refere to `reference-to-input-2` -echo "$FAST_WHY_DEPENDS_OUTPUT" | (! grep -q reference-to-input-2) -echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grep -q reference-to-input-2 +# But only the “precise” one should refer to `reference-to-input-2` +echo "$FAST_WHY_DEPENDS_OUTPUT" | grepQuietInverse reference-to-input-2 +echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grepQuiet reference-to-input-2 diff -Nru nix-2.8.0/.version nix-2.16.1+dfsg/.version --- nix-2.8.0/.version 2022-05-02 18:51:03.000000000 +0000 +++ nix-2.16.1+dfsg/.version 2023-06-06 10:55:23.000000000 +0000 @@ -1 +1 @@ -2.8.0 \ No newline at end of file +2.16.1