Provenance

doc/manual/source/protocols/json/schema/store-object-info-v2.yaml

@@ -179,6 +179,15 @@ $defs:
           The total size of this store object and every other object in its [closure](@docroot@/glossary.md#gloss-closure).
 
           > This field is not stored at all, but computed by traversing the other fields across all the store objects in a closure.
+
+      provenance:
+        oneOf:
+          - type: "null"
+          - type: object # FIXME
+        title: Provenance
+        description: |
+          An arbitrary JSON object containing provenance information about the store object, or `null` if not available.
+
     additionalProperties: false
 
   narInfo:
@@ -262,4 +271,13 @@ $defs:
           > This is an impure "`.narinfo`" field that may not be included in certain contexts.
 
           > This field is not stored at all, but computed by traversing the other fields across all the store objects in a closure.
+
+      provenance:
+        oneOf:
+          - type: "null"
+          - type: object # FIXME
+        title: Provenance
+        description: |
+          An arbitrary JSON object containing provenance information about the store object, or `null` if not available.
+
     additionalProperties: false

src/libcmd/include/nix/cmd/installable-flake.hh

@@ -72,6 +72,8 @@ struct InstallableFlake : InstallableValue
     ref<flake::LockedFlake> getLockedFlake() const;
 
     FlakeRef nixpkgsFlakeRef() const;
+
+    std::shared_ptr<const Provenance> makeProvenance(std::string_view attrPath) const;
 };
 
 /**

src/libcmd/installable-flake.cc

@@ -17,6 +17,7 @@
 #include "nix/util/url.hh"
 #include "nix/fetchers/registry.hh"
 #include "nix/store/build-result.hh"
+#include "nix/flake/provenance.hh"
 
 #include <regex>
 #include <queue>
@@ -84,6 +85,8 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
 
     auto attrPath = attr->getAttrPathStr();
 
+    PushProvenance pushedProvenance(*state, makeProvenance(attrPath));
+
     if (!attr->isDerivation()) {
 
         // FIXME: use eval cache?
@@ -172,6 +175,8 @@ std::vector<ref<eval_cache::AttrCursor>> InstallableFlake::getCursors(EvalState
     for (auto & attrPath : attrPaths) {
         debug("trying flake output attribute '%s'", attrPath);
 
+        PushProvenance pushedProvenance(state, makeProvenance(attrPath));
+
         auto attr = root->findAlongAttrPath(AttrPath::parse(state, attrPath));
         if (attr) {
             res.push_back(ref(*attr));
@@ -212,4 +217,12 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const
     return defaultNixpkgsFlakeRef();
 }
 
+std::shared_ptr<const Provenance> InstallableFlake::makeProvenance(std::string_view attrPath) const
+{
+    auto provenance = getLockedFlake()->flake.provenance;
+    if (!provenance)
+        return nullptr;
+    return std::make_shared<const FlakeProvenance>(provenance, std::string(attrPath));
+}
+
 } // namespace nix

src/libexpr/eval.cc

@@ -257,6 +257,8 @@ EvalMemory::EvalMemory()
     assertGCInitialized();
 }
 
+thread_local EvalState::EvalContext EvalState::evalContext;
+
 EvalState::EvalState(
     const LookupPath & lookupPathFromArguments,
     ref<Store> store,

src/libexpr/include/nix/expr/eval.hh

@@ -52,6 +52,7 @@ enum RepairFlag : bool;
 struct MemorySourceAccessor;
 struct MountedSourceAccessor;
 struct AsyncPathWriter;
+struct Provenance;
 
 namespace eval_cache {
 class EvalCache;
@@ -1128,6 +1129,20 @@ private:
     friend class ListBuilder;
 
 public:
+
+    /**
+     * Per-thread evaluation context. This context is propagated to worker threads when a value is evaluated
+     * asynchronously.
+     */
+    struct EvalContext
+    {
+        std::shared_ptr<const Provenance> provenance;
+
+        // FIXME: move callDepth here.
+    };
+
+    thread_local static EvalContext evalContext;
+
     /**
      * Worker threads manager.
      *
@@ -1173,6 +1188,24 @@ SourcePath resolveExprPath(SourcePath path, bool addDefaultNix = true);
  */
 bool isAllowedURI(std::string_view uri, const Strings & allowedPaths);
 
+struct PushProvenance
+{
+    EvalState & state;
+    std::shared_ptr<const Provenance> prev;
+
+    PushProvenance(EvalState & state, std::shared_ptr<const Provenance> prov)
+        : state(state)
+    {
+        state.evalContext.provenance.swap(prev);
+        state.evalContext.provenance.swap(prov);
+    }
+
+    ~PushProvenance()
+    {
+        state.evalContext.provenance.swap(prev);
+    }
+};
+
 } // namespace nix
 
 #include "nix/expr/eval-inline.hh"

src/libexpr/parallel-eval.cc

@@ -276,7 +276,12 @@ static void prim_parallel(EvalState & state, const PosIdx pos, Value ** args, Va
         std::vector<std::pair<Executor::work_t, uint8_t>> work;
         for (auto value : args[0]->listView())
             if (!value->isFinished())
-                work.emplace_back([value(allocRootValue(value)), &state, pos]() { state.forceValue(**value, pos); }, 0);
+                work.emplace_back(
+                    [value(allocRootValue(value)), &state, pos, evalContext(state.evalContext)]() {
+                        state.evalContext = evalContext;
+                        state.forceValue(**value, pos);
+                    },
+                    0);
         state.executor->spawn(std::move(work));
     }
 

src/libexpr/primops.cc

@@ -1847,7 +1847,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
     }
 
     /* Write the resulting term into the Nix store directory. */
-    auto drvPath = writeDerivation(*state.store, *state.asyncPathWriter, drv, state.repair);
+    auto drvPath =
+        writeDerivation(*state.store, *state.asyncPathWriter, drv, state.repair, false, state.evalContext.provenance);
     auto drvPathS = state.store->printStorePath(drvPath);
 
     printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
@@ -2733,7 +2734,8 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value ** args, Valu
                                                      ContentAddressMethod::Raw::Text,
                                                      HashAlgorithm::SHA256,
                                                      refs,
-                                                     state.repair);
+                                                     state.repair,
+                                                     state.evalContext.provenance);
                                              });
 
     /* Note: we don't need to add `context' to the context of the

src/libexpr/value-to-json.cc

@@ -30,7 +30,11 @@ static void parallelForceDeep(EvalState & state, Value & v, PosIdx pos)
             return;
         for (auto & a : *v.attrs())
             work.emplace_back(
-                [value(allocRootValue(a.value)), pos(a.pos), &state]() { parallelForceDeep(state, **value, pos); }, 0);
+                [value(allocRootValue(a.value)), pos(a.pos), &state, evalContext(state.evalContext)]() {
+                    state.evalContext = evalContext;
+                    parallelForceDeep(state, **value, pos);
+                },
+                0);
         break;
     }
 

src/libfetchers/attrs.cc

@@ -27,6 +27,9 @@ nlohmann::json attrsToJSON(const Attrs & attrs)
 {
     nlohmann::json json;
     for (auto & attr : attrs) {
+        /* The __final attribute is purely internal, so never serialize it. */
+        if (attr.first == "__final")
+            continue;
         if (auto v = std::get_if<uint64_t>(&attr.second)) {
             json[attr.first] = *v;
         } else if (auto v = std::get_if<std::string>(&attr.second)) {

src/libfetchers/fetchers.cc

@@ -4,7 +4,7 @@
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/util/json-utils.hh"
 #include "nix/fetchers/fetch-settings.hh"
-#include "nix/fetchers/fetch-to-store.hh"
+#include "nix/fetchers/provenance.hh"
 #include "nix/util/url.hh"
 #include "nix/util/forwarding-source-accessor.hh"
 #include "nix/util/archive.hh"
@@ -196,7 +196,6 @@ bool Input::contains(const Input & other) const
     return false;
 }
 
-// FIXME: remove
 std::tuple<StorePath, ref<SourceAccessor>, Input> Input::fetchToStore(const Settings & settings, Store & store) const
 {
     if (!scheme)
@@ -341,6 +340,8 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
                 {{"hash", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true)}});
         }
 
+        accessor->provenance = std::make_shared<TreeProvenance>(*this);
+
         // FIXME: ideally we would use the `showPath()` of the
         // "real" accessor for this fetcher type.
         accessor->setPathDisplay("«" + to_string(true) + "»");
@@ -365,6 +366,8 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
         else
             accessor->fingerprint = result.getFingerprint(store);
 
+        accessor->provenance = std::make_shared<TreeProvenance>(result);
+
         return {accessor, std::move(result)};
     } catch (Error & e) {
         if (storePath) {

src/libfetchers/filtering-source-accessor.cc

@@ -68,6 +68,13 @@ std::pair<CanonPath, std::optional<std::string>> FilteringSourceAccessor::getFin
     return next->getFingerprint(prefix / path);
 }
 
+std::shared_ptr<const Provenance> FilteringSourceAccessor::getProvenance(const CanonPath & path)
+{
+    if (provenance)
+        return SourceAccessor::getProvenance(path);
+    return next->getProvenance(prefix / path);
+}
+
 void FilteringSourceAccessor::invalidateCache(const CanonPath & path)
 {
     next->invalidateCache(prefix / path);

src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh

@@ -52,6 +52,8 @@ struct FilteringSourceAccessor : SourceAccessor
 
     std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override;
 
+    std::shared_ptr<const Provenance> getProvenance(const CanonPath & path) override;
+
     void invalidateCache(const CanonPath & path) override;
 
     /**

src/libfetchers/include/nix/fetchers/meson.build

@@ -10,6 +10,7 @@ headers = files(
   'git-lfs-fetch.hh',
   'git-utils.hh',
   'input-cache.hh',
+  'provenance.hh',
   'registry.hh',
   'tarball.hh',
 )

src/libfetchers/include/nix/fetchers/provenance.hh

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "nix/util/provenance.hh"
+#include "nix/fetchers/fetchers.hh"
+
+namespace nix {
+
+struct TreeProvenance : Provenance
+{
+    ref<nlohmann::json> attrs;
+
+    TreeProvenance(const fetchers::Input & input);
+
+    nlohmann::json to_json() const override;
+};
+
+} // namespace nix

src/libfetchers/meson.build

@@ -49,6 +49,7 @@ sources = files(
   'input-cache.cc',
   'mercurial.cc',
   'path.cc',
+  'provenance.cc',
   'registry.cc',
   'tarball.cc',
 )

src/libfetchers/provenance.cc

@@ -0,0 +1,27 @@
+#include "nix/fetchers/provenance.hh"
+#include "nix/fetchers/attrs.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+TreeProvenance::TreeProvenance(const fetchers::Input & input)
+    : attrs(make_ref<nlohmann::json>([&]() {
+        // Remove the narHash attribute from the provenance info, as it's redundant (it's already recorded in the store
+        // path info).
+        auto attrs2 = input.attrs;
+        attrs2.erase("narHash");
+        return fetchers::attrsToJSON(attrs2);
+    }()))
+{
+}
+
+nlohmann::json TreeProvenance::to_json() const
+{
+    return nlohmann::json{
+        {"type", "tree"},
+        {"attrs", *attrs},
+    };
+}
+
+} // namespace nix

src/libfetchers/tarball.cc

@@ -9,9 +9,30 @@
 #include "nix/store/store-api.hh"
 #include "nix/fetchers/git-utils.hh"
 #include "nix/fetchers/fetch-settings.hh"
+#include "nix/util/provenance.hh"
+
+#include <nlohmann/json.hpp>
 
 namespace nix::fetchers {
 
+struct FetchurlProvenance : Provenance
+{
+    std::string url;
+
+    FetchurlProvenance(const std::string & url)
+        : url(url)
+    {
+    }
+
+    nlohmann::json to_json() const override
+    {
+        return nlohmann::json{
+            {"type", "fetchurl"},
+            {"url", url},
+        };
+    }
+};
+
 DownloadFileResult downloadFile(
     Store & store,
     const Settings & settings,
@@ -83,6 +104,13 @@ DownloadFileResult downloadFile(
             },
             hashString(HashAlgorithm::SHA256, sink.s));
         info.narSize = sink.s.size();
+        if (experimentalFeatureSettings.isEnabled(Xp::Provenance)) {
+            auto sanitizedUrl = request.uri.parsed();
+            if (sanitizedUrl.authority)
+                sanitizedUrl.authority->password.reset();
+            sanitizedUrl.query.clear();
+            info.provenance = std::make_shared<FetchurlProvenance>(sanitizedUrl.to_string());
+        }
         auto source = StringSource{sink.s};
         store.addToStore(info, source, NoRepair, NoCheckSigs);
         storePath = std::move(info.path);

src/libflake/flake.cc

@@ -269,6 +269,7 @@ static Flake readFlake(
         .resolvedRef = resolvedRef,
         .lockedRef = lockedRef,
         .path = flakePath,
+        .provenance = flakePath.getProvenance(),
     };
 
     if (auto description = vInfo.attrs()->get(state.s.description)) {

src/libflake/include/nix/flake/flake.hh

@@ -10,6 +10,7 @@
 namespace nix {
 
 class EvalState;
+struct Provenance;
 
 namespace flake {
 
@@ -94,6 +95,11 @@ struct Flake
      */
     SourcePath path;
 
+    /**
+     * Cached provenance of `flake.nix` (equivalent to `path.getProvenance()`).
+     */
+    std::shared_ptr<const Provenance> provenance;
+
     /**
      * Pretend that `lockedRef` is dirty.
      */

src/libflake/include/nix/flake/meson.build

@@ -6,6 +6,7 @@ headers = files(
   'flake.hh',
   'flakeref.hh',
   'lockfile.hh',
+  'provenance.hh',
   'settings.hh',
   'url-name.hh',
 )