From d45638f19617da9daa586b07cd20bcc9e9161193 Mon Sep 17 00:00:00 2001 From: Beojan Stanislaus Date: Fri, 23 Jan 2026 09:22:59 -0800 Subject: [PATCH 1/7] product_query changes Won't build yet --- phlex/core/product_query.cpp | 35 +++----------- phlex/core/product_query.hpp | 92 +++++++++++++++++++++++++----------- 2 files changed, 72 insertions(+), 55 deletions(-) diff --git a/phlex/core/product_query.cpp b/phlex/core/product_query.cpp index 5f442e70..78c36f57 100644 --- a/phlex/core/product_query.cpp +++ b/phlex/core/product_query.cpp @@ -2,37 +2,16 @@ #include "fmt/format.h" -#include - namespace phlex { - product_query::product_query() = default; - - product_query::product_query(experimental::product_specification spec, std::string layer) : - spec_{std::move(spec)}, layer_{std::move(layer)} - { - } - - product_query experimental::product_tag::operator()(std::string data_layer) && - { - if (data_layer.empty()) { - throw std::runtime_error("Cannot specify the empty string as a data layer."); - } - return {std::move(spec), std::move(data_layer)}; - } - + void product_query::set_type(experimental::type_id&& type) { + type_id_ = std::move(type); + } + std::string product_query::to_string() const { - if (layer_.empty()) { - return spec_.full(); - } - return fmt::format("{} ϵ {}", spec_.full(), layer_); - } - - experimental::product_tag operator""_in(char const* product_name, std::size_t length) - { - if (length == 0ull) { - throw std::runtime_error("Cannot specify product with empty name."); + if (suffix_) { + return fmt::format("{}/{} ϵ {}", creator_, *suffix_, layer_); } - return {experimental::product_specification::create(product_name)}; + return fmt::format("{} ϵ {}", creator_, layer_); } } diff --git a/phlex/core/product_query.hpp b/phlex/core/product_query.hpp index 55cc3956..f9042326 100644 --- a/phlex/core/product_query.hpp +++ b/phlex/core/product_query.hpp @@ -1,47 +1,90 @@ #ifndef PHLEX_CORE_PRODUCT_QUERY_HPP #define PHLEX_CORE_PRODUCT_QUERY_HPP -#include "phlex/model/product_specification.hpp" +#include "phlex/model/type_id.hpp" -// #include +#include +#include #include #include +// This allows optionals to be initialized using ""s syntax +using std::string_literals::operator""s; namespace phlex { + namespace detail { + template + requires std::is_same_v // has to be a template for static_assert(false) + class required_creator_name { + public: + consteval required_creator_name() + { + static_assert(false, "The creator name has not been set in this product_query."); + } + required_creator_name(T&& rhs) : content_(std::forward(rhs)) {} + + required_creator_name(std::string_view rhs) : content_(rhs) {} + + T&& release() {return std::move(content_);} + + private: + std::string content_; + }; + + template + requires std::is_same_v // has to be a template for static_assert(false) + class required_layer_name { + public: + consteval required_layer_name() + { + static_assert(false, "The layer name has not been set in this product_query."); + } + required_layer_name(T&& rhs) : content_(std::forward(rhs)) {} + + required_layer_name(std::string_view rhs) : content_(rhs) {} + + T&& release() {return std::move(content_);} + + private: + std::string content_; + }; + } + + struct product_tag { + detail::required_creator_name creator; + detail::required_layer_name layer; + std::optional suffix; + std::optional stage; + }; + class product_query { public: - // FIXME: Boost JSON's parameter retrieval facilities require a default constructor - // whenever the type is (e.g.) std::array. - product_query(); - product_query(experimental::product_specification spec, std::string layer); - - auto const& spec() const noexcept { return spec_; } - auto const& layer() const noexcept { return layer_; } - void set_type(experimental::type_id&& type) { spec_.set_type(std::move(type)); } + product_query() = default; // Required by boost JSON + product_query(product_tag&& tag) : creator_(tag.creator.release()), layer_(tag.layer.release()), suffix_(std::move(tag.suffix)), stage_(std::move(tag.stage)) {} + void set_type(experimental::type_id&& type); + std::string const& creator() const noexcept {return creator_;} + std::string const& layer() const noexcept {return layer_;} + std::optional const& suffix() const noexcept {return suffix_;} + std::optional const& stage() const noexcept {return stage_;} + experimental::type_id const& type() const noexcept {return type_id_;} std::string to_string() const; - auto operator<=>(product_query const&) const = default; - private: - experimental::product_specification spec_; + std::string creator_; std::string layer_; + std::optional suffix_; + std::optional stage_; + experimental::type_id type_id_; }; using product_queries = std::vector; -} - -namespace phlex::experimental { - struct product_tag { - product_specification spec; - product_query operator()(std::string layer) &&; - }; + std::ostream& operator<<(std::ostream& os, product_query const& label); namespace detail { // C is a container of product_queries template requires std::is_same_v::value_type, product_query> && - is_tuple::value + experimental::is_tuple::value class product_queries_type_setter {}; template class product_queries_type_setter> { @@ -66,16 +109,11 @@ namespace phlex::experimental { template requires std::is_same_v::value_type, product_query> && - is_tuple::value + experimental::is_tuple::value void populate_types(C& container) { detail::product_queries_type_setter populate_types{}; populate_types(container); } } - -namespace phlex { - experimental::product_tag operator""_in(char const* str, std::size_t); -} - #endif // PHLEX_CORE_PRODUCT_QUERY_HPP From bb44f6baf847bc061ea00a4e58f109a23f106534 Mon Sep 17 00:00:00 2001 From: Beojan Stanislaus Date: Mon, 26 Jan 2026 10:56:22 -0800 Subject: [PATCH 2/7] Part 1(!!!) of change to allow specifying creator This handles the API change, and ports the existing creator name check to work with the new product_query. --- CMakePresets.json | 2 +- phlex/configuration.cpp | 57 +++++++++++++++- phlex/core/detail/filter_impl.cpp | 8 +-- phlex/core/edge_creation_policy.cpp | 40 +++++------ phlex/core/input_arguments.cpp | 32 +++------ phlex/core/product_query.cpp | 53 +++++++++++++-- phlex/core/product_query.hpp | 50 ++++++++++---- phlex/model/type_id.hpp | 11 +++ plugins/CMakeLists.txt | 2 +- test/CMakeLists.txt | 2 +- test/allowed_families.cpp | 18 +++-- test/benchmarks/benchmark-04.jsonnet | 40 ++++++----- test/benchmarks/benchmark-05.jsonnet | 10 +++ test/benchmarks/benchmark-06.jsonnet | 58 +++++++++------- test/benchmarks/benchmark-07.jsonnet | 68 +++++++++++-------- test/benchmarks/benchmark-08.jsonnet | 6 +- test/benchmarks/benchmark-09.jsonnet | 6 +- test/benchmarks/benchmarks_provider.cpp | 2 +- test/benchmarks/last_index.cpp | 2 +- test/benchmarks/plus_101.cpp | 4 +- test/benchmarks/plus_one.cpp | 4 +- test/benchmarks/read_id.cpp | 3 +- test/benchmarks/verify_difference.cpp | 3 +- test/cached_execution.cpp | 21 +++--- test/class_registration.cpp | 13 ++-- test/configuration.cpp | 13 ++-- test/demo-giantdata/unfold_transform_fold.cpp | 8 +-- test/different_hierarchies.cpp | 10 +-- test/filter.cpp | 63 +++++++++++------ test/filter_impl.cpp | 6 +- test/fold.cpp | 15 ++-- test/framework_graph.cpp | 4 +- test/function_registration.cpp | 13 ++-- test/hierarchical_nodes.cpp | 17 +++-- test/max-parallelism/check_parallelism.cpp | 3 +- test/max-parallelism/provide_parallelism.cpp | 3 +- test/memory-checks/many_events.cpp | 4 +- test/mock-workflow/G4Stage1.libsonnet | 2 +- test/mock-workflow/G4Stage2.libsonnet | 4 +- test/mock-workflow/event_product.libsonnet | 10 ++- test/mock-workflow/id_provider.cpp | 2 +- test/multiple_function_registration.cpp | 26 ++++--- test/plugins/ij_source.cpp | 4 +- test/plugins/module.cpp | 5 +- test/product_query.cpp | 13 ++-- test/provider_test.cpp | 6 +- test/python/source.cpp | 4 +- test/type_distinction.cpp | 40 +++++++---- test/unfold.cpp | 20 +++--- test/vector_of_abstract_types.cpp | 8 ++- 50 files changed, 529 insertions(+), 289 deletions(-) diff --git a/CMakePresets.json b/CMakePresets.json index 79b4a20a..dc1d73c2 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -5,7 +5,7 @@ "hidden": false, "cacheVariables": { "CMAKE_EXPORT_COMPILE_COMMANDS": "YES", - "CMAKE_CXX_STANDARD": "20", + "CMAKE_CXX_STANDARD": "23", "CMAKE_CXX_STANDARD_REQUIRED": "YES", "CMAKE_CXX_EXTENSIONS": "NO", "CMAKE_PROJECT_TOP_LEVEL_INCLUDES": "CetProvideDependency" diff --git a/phlex/configuration.cpp b/phlex/configuration.cpp index 03cb6daa..b120ffc7 100644 --- a/phlex/configuration.cpp +++ b/phlex/configuration.cpp @@ -1,8 +1,53 @@ #include "phlex/configuration.hpp" #include "phlex/core/product_query.hpp" -#include +#include #include +#include + +namespace { + std::optional value_if_exists(boost::json::object const& obj, + std::string_view parameter) + { + if (!obj.contains(parameter)) { + return std::nullopt; + } + auto const& val = obj.at(parameter); + if (!val.is_string()) { + std::string kind; + switch (val.kind()) { + case boost::json::kind::null: + // seems reasonable to interpret this as if the value were not provided + return std::nullopt; + break; + case boost::json::kind::bool_: + kind = "bool"s; + break; + case boost::json::kind::int64: + kind = "std::int64_t"s; + break; + case boost::json::kind::uint64: + kind = "std::uint64_t"s; + break; + case boost::json::kind::double_: + kind = "double"s; + break; + case boost::json::kind::array: + kind = "array"s; + break; + case boost::json::kind::object: + kind = "object"s; + break; + default: + kind = "HOW??"s; + break; + } + throw std::runtime_error(fmt::format( + "Error retrieving parameter '{}'. Should be a string but is instead a {}", parameter, kind)); + } + return std::string(val.get_string()); + } +} namespace phlex { std::vector configuration::keys() const @@ -25,7 +70,13 @@ namespace phlex { { using detail::value_decorate_exception; auto query_object = jv.as_object(); - return product_query{{value_decorate_exception(query_object, "product")}, - value_decorate_exception(query_object, "layer")}; + auto creator = value_decorate_exception(query_object, "creator"); + auto layer = value_decorate_exception(query_object, "layer"); + auto suffix = value_if_exists(query_object, "suffix"); + auto stage = value_if_exists(query_object, "stage"); + return product_tag{.creator = std::move(creator), + .layer = std::move(layer), + .suffix = std::move(suffix), + .stage = std::move(stage)}; } } diff --git a/phlex/core/detail/filter_impl.cpp b/phlex/core/detail/filter_impl.cpp index a7156cc9..5d522af6 100644 --- a/phlex/core/detail/filter_impl.cpp +++ b/phlex/core/detail/filter_impl.cpp @@ -5,12 +5,8 @@ #include namespace { - phlex::product_query const output_dummy{ - phlex::experimental::product_specification{ - phlex::experimental::algorithm_name{"for_output_only", ""}, - "for_output_only", - phlex::experimental::type_id{}}, - "dummy_layer"}; + phlex::product_query const output_dummy = phlex::product_tag{ + .creator = "for_output_only"s, .layer = "dummy_layer"s, .suffix = "for_output_only"s}; phlex::product_queries const for_output_only{output_dummy}; } diff --git a/phlex/core/edge_creation_policy.cpp b/phlex/core/edge_creation_policy.cpp index 1ab4ab71..22924b6b 100644 --- a/phlex/core/edge_creation_policy.cpp +++ b/phlex/core/edge_creation_policy.cpp @@ -9,55 +9,55 @@ namespace phlex::experimental { edge_creation_policy::named_output_port const* edge_creation_policy::find_producer( product_query const& query) const { - auto const& spec = query.spec(); - auto [b, e] = producers_.equal_range(spec.name()); + // TODO: Update later with correct querying + auto [b, e] = producers_.equal_range(query.suffix().value_or("")); if (b == e) { spdlog::debug( "Failed to find an algorithm that creates {} products. Assuming it comes from a provider", - spec.name()); + query.suffix().value_or("\"\"")); return nullptr; } std::map candidates; for (auto const& [key, producer] : std::ranges::subrange{b, e}) { - if (producer.node.match(spec.qualifier())) { - if (spec.type() != producer.type) { - spdlog::debug("Matched {} ({}) from {} but types don't match (`{}` vs `{}`). Excluding " + // TODO: Definitely not right yet + if (producer.node.plugin() == query.creator() || + producer.node.algorithm() == query.creator()) { + if (query.type() != producer.type) { + spdlog::debug("Matched ({}) from {} but types don't match (`{}` vs `{}`). Excluding " "from candidate list.", - spec.full(), query.to_string(), producer.node.full(), - spec.type(), + query.type(), producer.type); } else { - if (spec.type().exact_compare(producer.type)) { - spdlog::debug("Matched {} ({}) from {} and types match. Keeping in candidate list.", - spec.full(), + if (query.type().exact_compare(producer.type)) { + spdlog::debug("Matched ({}) from {} and types match. Keeping in candidate list.", query.to_string(), producer.node.full()); } else { - spdlog::warn("Matched {} ({}) from {} and types match, but not exactly (produce {} and " + spdlog::warn("Matched ({}) from {} and types match, but not exactly (produce {} and " "consume {}). Keeping in candidate list!", - spec.full(), query.to_string(), producer.node.full(), - spec.type().exact_name(), + query.type().exact_name(), producer.type.exact_name()); } candidates.emplace(producer.node.full(), &producer); } } + else { + spdlog::error("Creator name mismatch between ({}) and {}", query.to_string(), producer.node.full()); + } } if (candidates.empty()) { - throw std::runtime_error("Cannot identify product matching the specified label " + - spec.full()); + throw std::runtime_error("Cannot identify product matching the query " + query.to_string()); } if (candidates.size() > 1ull) { - std::string msg = - fmt::format("More than one candidate matches the specification {}: \n - {}\n", - spec.full(), - fmt::join(std::views::keys(candidates), "\n - ")); + std::string msg = fmt::format("More than one candidate matches the query {}: \n - {}\n", + query.to_string(), + fmt::join(std::views::keys(candidates), "\n - ")); throw std::runtime_error(msg); } diff --git a/phlex/core/input_arguments.cpp b/phlex/core/input_arguments.cpp index d8d86011..ad5faa97 100644 --- a/phlex/core/input_arguments.cpp +++ b/phlex/core/input_arguments.cpp @@ -2,32 +2,22 @@ #include "fmt/format.h" -#include -#include #include -#include namespace phlex::experimental::detail { void verify_no_duplicate_input_products(std::string const& algorithm_name, - product_queries to_sort) + product_queries input_queries) { - std::sort(begin(to_sort), end(to_sort)); - std::set unique_and_sorted(begin(to_sort), end(to_sort)); - product_queries duplicates; - std::set_difference(begin(to_sort), - end(to_sort), - begin(unique_and_sorted), - end(unique_and_sorted), - std::back_inserter(duplicates)); - if (empty(duplicates)) { - return; + for (std::size_t i = 0; i < input_queries.size(); ++i) { + for (std::size_t j = i + 1; j < input_queries.size(); ++j) { + if (input_queries[i].match(input_queries[j]) || input_queries[j].match(input_queries[i])) { + throw std::runtime_error( + fmt::format("The algorithm {} has at least one duplicate input:\n- {}\n- {}\n", + algorithm_name, + input_queries[i].to_string(), + input_queries[j].to_string())); + } + } } - - std::string error = - fmt::format("Algorithm '{}' uses the following products more than once:\n", algorithm_name); - for (auto const& label : duplicates) { - error += fmt::format(" - '{}'\n", label.to_string()); - } - throw std::runtime_error(error); } } diff --git a/phlex/core/product_query.cpp b/phlex/core/product_query.cpp index 78c36f57..edafd118 100644 --- a/phlex/core/product_query.cpp +++ b/phlex/core/product_query.cpp @@ -3,10 +3,47 @@ #include "fmt/format.h" namespace phlex { - void product_query::set_type(experimental::type_id&& type) { - type_id_ = std::move(type); - } - + void product_query::set_type(experimental::type_id&& type) { type_id_ = std::move(type); } + + // Check that all products selected by /other/ would satisfy this query + bool product_query::match(product_query const& other) const + { + if (creator_ != other.creator_) { + return false; + } + if (layer_ != other.layer_) { + return false; + } + if (suffix_ && suffix_ != other.suffix_) { + return false; + } + if (stage_ && stage_ != other.stage_) { + return false; + } + // Special case. If other has an unset type_id, ignore this in case it just hasn't been set yet + if (type_id_.valid() && other.type_id_.valid() && type_id_ != other.type_id_) { + return false; + } + return true; + } + + // Check if a product_specification satisfies this query + bool product_query::match(experimental::product_specification const& spec) const + { + if (creator_ != spec.algorithm()) { + return false; + } + if (type_id_ != spec.type()) { + return false; + } + if (suffix_) { + if (*suffix_ != spec.name()) { + return false; + } + } + return true; + } + std::string product_query::to_string() const { if (suffix_) { @@ -14,4 +51,12 @@ namespace phlex { } return fmt::format("{} ϵ {}", creator_, layer_); } + + experimental::product_specification product_query::spec() const + { + if (!suffix()) { + throw std::logic_error("Product suffixes are (temporarily) mandatory"); + } + return experimental::product_specification::create(*suffix()); + } } diff --git a/phlex/core/product_query.hpp b/phlex/core/product_query.hpp index f9042326..a049aaae 100644 --- a/phlex/core/product_query.hpp +++ b/phlex/core/product_query.hpp @@ -1,9 +1,9 @@ #ifndef PHLEX_CORE_PRODUCT_QUERY_HPP #define PHLEX_CORE_PRODUCT_QUERY_HPP +#include "phlex/model/product_specification.hpp" #include "phlex/model/type_id.hpp" -#include #include #include #include @@ -24,7 +24,7 @@ namespace phlex { required_creator_name(std::string_view rhs) : content_(rhs) {} - T&& release() {return std::move(content_);} + T&& release() { return std::move(content_); } private: std::string content_; @@ -42,7 +42,7 @@ namespace phlex { required_layer_name(std::string_view rhs) : content_(rhs) {} - T&& release() {return std::move(content_);} + T&& release() { return std::move(content_); } private: std::string content_; @@ -55,20 +55,46 @@ namespace phlex { std::optional suffix; std::optional stage; }; - + class product_query { public: product_query() = default; // Required by boost JSON - product_query(product_tag&& tag) : creator_(tag.creator.release()), layer_(tag.layer.release()), suffix_(std::move(tag.suffix)), stage_(std::move(tag.stage)) {} + product_query(product_tag&& tag) : + creator_(tag.creator.release()), + layer_(tag.layer.release()), + suffix_(std::move(tag.suffix)), + stage_(std::move(tag.stage)) + { + if (creator_.empty()) { + throw std::runtime_error("Cannot specify product with empty creator name."); + } + if (layer_.empty()) { + throw std::runtime_error("Cannot specify the empty string as a data layer."); + } + } void set_type(experimental::type_id&& type); - std::string const& creator() const noexcept {return creator_;} - std::string const& layer() const noexcept {return layer_;} - std::optional const& suffix() const noexcept {return suffix_;} - std::optional const& stage() const noexcept {return stage_;} - experimental::type_id const& type() const noexcept {return type_id_;} + std::string const& creator() const noexcept { return creator_; } + std::string const& layer() const noexcept { return layer_; } + std::optional const& suffix() const noexcept { return suffix_; } + std::optional const& stage() const noexcept { return stage_; } + experimental::type_id const& type() const noexcept { return type_id_; } + + // Check that all products selected by /other/ would satisfy this query + bool match(product_query const& other) const; + + // Check if a product_specification satisfies this query + bool match(experimental::product_specification const& spec) const; + std::string to_string() const; + // temporary additional members for transition + experimental::product_specification spec() const; + bool operator==(product_query const& rhs) const + { + return this->spec() == rhs.spec() && this->layer_ == rhs.layer_; + } + private: std::string creator_; std::string layer_; @@ -78,8 +104,6 @@ namespace phlex { }; using product_queries = std::vector; - std::ostream& operator<<(std::ostream& os, product_query const& label); - namespace detail { // C is a container of product_queries template @@ -94,7 +118,7 @@ namespace phlex { template void set_type(C& container) { - container.at(index_).set_type(make_type_id()); + container.at(index_).set_type(experimental::make_type_id()); ++index_; } diff --git a/phlex/model/type_id.hpp b/phlex/model/type_id.hpp index ec6399dd..d7ce943e 100644 --- a/phlex/model/type_id.hpp +++ b/phlex/model/type_id.hpp @@ -7,6 +7,7 @@ #include "fmt/format.h" #include "fmt/ranges.h" #include +#include #include #include @@ -43,6 +44,16 @@ namespace phlex::experimental { constexpr builtin fundamental() const { return static_cast(id_ & 0x0F); } + template + friend constexpr void tag_invoke( + boost::hash2::hash_append_tag const&, Provider const&, Hash& h, Flavor const& f, type_id const* v) + { + boost::hash2::hash_append(h, f, v->id_); + if (v->has_children()) { + boost::hash2::hash_append(h, f, v->children_); + } + } + constexpr std::strong_ordering operator<=>(type_id const& rhs) const { // This ordering is arbitrary but defined diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 63a32a9e..af71df3f 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -1,7 +1,7 @@ # Phlex provided core plugins # plugin for running Python algorithms in phlex -add_subdirectory(python) +# add_subdirectory(python) add_library(layer_generator layer_generator.cpp) target_link_libraries(layer_generator PRIVATE phlex::core) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ab51b55f..d0b8e237 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -236,7 +236,7 @@ add_subdirectory(plugins) add_subdirectory(utilities) add_subdirectory(mock-workflow) add_subdirectory(demo-giantdata) -add_subdirectory(python) +# add_subdirectory(python) if(PHLEX_USE_FORM) add_subdirectory(form) diff --git a/test/allowed_families.cpp b/test/allowed_families.cpp index d0f3a54f..514affd0 100644 --- a/test/allowed_families.cpp +++ b/test/allowed_families.cpp @@ -43,16 +43,22 @@ TEST_CASE("Testing families", "[data model]") // Wire up providers for each level g.provide("run_id_provider", provide_index, concurrency::unlimited) - .output_product("id"_in("run")); + .output_product(product_query({.creator = "dummy"s, .layer = "run"s, .suffix = "id"s})); g.provide("subrun_id_provider", provide_index, concurrency::unlimited) - .output_product("id"_in("subrun")); + .output_product(product_query({.creator = "dummy"s, .layer = "subrun"s, .suffix = "id"s})); g.provide("event_id_provider", provide_index, concurrency::unlimited) - .output_product("id"_in("event")); + .output_product(product_query({.creator = "dummy"s, .layer = "event"s, .suffix = "id"s})); - g.observe("se", check_two_ids).input_family("id"_in("subrun"), "id"_in("event")); - g.observe("rs", check_two_ids).input_family("id"_in("run"), "id"_in("subrun")); + g.observe("se", check_two_ids) + .input_family(product_query({.creator = "dummy"s, .layer = "subrun"s, .suffix = "id"s}), + product_query({.creator = "dummy"s, .layer = "event"s, .suffix = "id"s})); + g.observe("rs", check_two_ids) + .input_family(product_query({.creator = "dummy"s, .layer = "run"s, .suffix = "id"s}), + product_query({.creator = "dummy"s, .layer = "subrun"s, .suffix = "id"s})); g.observe("rse", check_three_ids) - .input_family("id"_in("run"), "id"_in("subrun"), "id"_in("event")); + .input_family(product_query({.creator = "dummy"s, .layer = "run"s, .suffix = "id"s}), + product_query({.creator = "dummy"s, .layer = "subrun"s, .suffix = "id"s}), + product_query({.creator = "dummy"s, .layer = "event"s, .suffix = "id"s})); g.execute(); CHECK(g.execution_counts("se") == 1ull); diff --git a/test/benchmarks/benchmark-04.jsonnet b/test/benchmarks/benchmark-04.jsonnet index 5d5a3702..18f5287d 100644 --- a/test/benchmarks/benchmark-04.jsonnet +++ b/test/benchmarks/benchmark-04.jsonnet @@ -1,22 +1,26 @@ { - driver: { - cpp: 'generate_layers', - layers: { - event: { total: 100000 } - } - }, - sources: { - provider: { - cpp: 'benchmarks_provider' - } - }, - modules: { - a_creator: { - cpp: 'last_index', + "driver": { + "cpp": "generate_layers", + "layers": { + "event": { "total": 100000 } + } }, - read_index: { - cpp: 'read_index', - consumes: { product: 'a', layer: "event" } + "sources": { + "provider": { + "cpp": "benchmarks_provider" + } }, - }, + "modules": { + "a_creator": { + "cpp": "last_index" + }, + "read_index": { + "cpp": "read_index", + "consumes": { + "creator": "a_creator", + "suffix": "a", + "layer": "event" + } + } + } } diff --git a/test/benchmarks/benchmark-05.jsonnet b/test/benchmarks/benchmark-05.jsonnet index 495995e1..9559ea5b 100644 --- a/test/benchmarks/benchmark-05.jsonnet +++ b/test/benchmarks/benchmark-05.jsonnet @@ -21,6 +21,16 @@ }, d: { cpp: 'verify_difference', + i: { + creator: 'b_creator', + layer: 'event', + suffix: 'b' + }, + j: { + creator: 'c_creator', + layer: 'event', + suffix: 'c' + }, expected: 0 }, }, diff --git a/test/benchmarks/benchmark-06.jsonnet b/test/benchmarks/benchmark-06.jsonnet index ee6c9a50..879d9c1b 100644 --- a/test/benchmarks/benchmark-06.jsonnet +++ b/test/benchmarks/benchmark-06.jsonnet @@ -1,27 +1,39 @@ { - driver: { - cpp: 'generate_layers', - layers: { - event: { total: 100000 } - } - }, - sources: { - provider: { - cpp: 'benchmarks_provider' - } - }, - modules: { - a_creator: { - cpp: 'last_index', - }, - b_creator: { - cpp: 'plus_one', + "driver": { + "cpp": "generate_layers", + "layers": { + "event": { "total": 100000 } + } }, - c_creator: { - cpp: 'plus_101', + "sources": { + "provider": { + "cpp": "benchmarks_provider" + } }, - d: { - cpp: 'verify_difference', - }, - }, + "modules": { + "a_creator": { + "cpp": "last_index" + }, + "b_creator": { + "cpp": "plus_one", + "input": { "creator": "a_creator", "layer": "event", "suffix": "a" } + }, + "c_creator": { + "cpp": "plus_101", + "input": { "creator": "a_creator", "layer": "event", "suffix": "a" } + }, + "d": { + "cpp": "verify_difference", + "i": { + "creator": "b_creator", + "layer": "event", + "suffix": "b" + }, + "j": { + "creator": "c_creator", + "layer": "event", + "suffix": "c" + } + } + } } diff --git a/test/benchmarks/benchmark-07.jsonnet b/test/benchmarks/benchmark-07.jsonnet index 3b6eef46..026f9b72 100644 --- a/test/benchmarks/benchmark-07.jsonnet +++ b/test/benchmarks/benchmark-07.jsonnet @@ -1,33 +1,43 @@ { - driver: { - cpp: 'generate_layers', - layers: { - event: { total: 100000 } - } - }, - sources: { - provider: { - cpp: 'benchmarks_provider' - } - }, - modules: { - even_filter: { - cpp: 'accept_even_ids', - input: { product: 'id', layer: 'event' }, - }, - b_creator: { - cpp: 'last_index', - experimental_when: ['even_filter:accept_even_ids'], - produces: 'b', + "driver": { + "cpp": "generate_layers", + "layers": { + "event": { "total": 100000 } + } }, - c_creator: { - cpp: 'last_index', - experimental_when: ['even_filter:accept_even_ids'], - produces: 'c', + "sources": { + "provider": { + "cpp": "benchmarks_provider" + } }, - d: { - cpp: 'verify_difference', - expected: 0 - }, - }, + "modules": { + "even_filter": { + "cpp": "accept_even_ids", + "input": { "creator": "input", "suffix": "id", "layer": "event" } + }, + "b_creator": { + "cpp": "last_index", + "experimental_when": ["even_filter:accept_even_ids"], + "produces": "b" + }, + "c_creator": { + "cpp": "last_index", + "experimental_when": ["even_filter:accept_even_ids"], + "produces": "c" + }, + "d": { + "cpp": "verify_difference", + "i": { + "creator": "b_creator", + "layer": "event", + "suffix": "b" + }, + "j": { + "creator": "c_creator", + "layer": "event", + "suffix": "c" + }, + "expected": 0 + } + } } diff --git a/test/benchmarks/benchmark-08.jsonnet b/test/benchmarks/benchmark-08.jsonnet index a3f30291..67d38be2 100644 --- a/test/benchmarks/benchmark-08.jsonnet +++ b/test/benchmarks/benchmark-08.jsonnet @@ -19,17 +19,17 @@ local max_number = 100000; }, even_filter: { cpp: 'accept_even_numbers', - consumes: { product: 'a', layer: 'event' } + consumes: { creator: 'a_creator', suffix: 'a', layer: 'event' } }, fibonacci_filter: { cpp: 'accept_fibonacci_numbers', - consumes: { product: 'a', layer: "event" }, + consumes: { creator: 'a_creator', suffix: 'a', layer: "event" }, max_number: max_number, }, d: { cpp: 'verify_even_fibonacci_numbers', experimental_when: ['even_filter:accept_even_numbers', 'fibonacci_filter:accept'], - consumes: { product: 'a', layer: "event" }, + consumes: { creator: 'a_creator', suffix: 'a', layer: "event" }, max_number: max_number, }, }, diff --git a/test/benchmarks/benchmark-09.jsonnet b/test/benchmarks/benchmark-09.jsonnet index b70f69cb..d026935c 100644 --- a/test/benchmarks/benchmark-09.jsonnet +++ b/test/benchmarks/benchmark-09.jsonnet @@ -13,18 +13,20 @@ modules: { a_creator: { cpp: 'last_index', + input: {creator: "a_creator", layer: "event", suffix: "a"} }, b_creator: { cpp: 'plus_one', + input: {creator: "a_creator", layer: "event", suffix: "a"} }, even_filter: { cpp: 'accept_even_numbers', - consumes: { product: 'a', layer: "event" } + consumes: { creator: 'a_creator', suffix: 'a', layer: "event" } }, d: { cpp: 'read_index', experimental_when: ['even_filter:accept_even_numbers'], - consumes: { product: 'b', layer: "event" } + consumes: { creator: 'b_creator', suffix: 'b', layer: "event" } }, }, } diff --git a/test/benchmarks/benchmarks_provider.cpp b/test/benchmarks/benchmarks_provider.cpp index 126bd676..704e536c 100644 --- a/test/benchmarks/benchmarks_provider.cpp +++ b/test/benchmarks/benchmarks_provider.cpp @@ -4,5 +4,5 @@ PHLEX_REGISTER_PROVIDERS(s) { using namespace phlex; s.provide("provide_id", [](data_cell_index const& id) { return id; }) - .output_product("id"_in("event")); + .output_product(product_tag{.creator = "input"s, .layer = "event"s, .suffix = "id"s}); } diff --git a/test/benchmarks/last_index.cpp b/test/benchmarks/last_index.cpp index bc1c6474..7fb34da8 100644 --- a/test/benchmarks/last_index.cpp +++ b/test/benchmarks/last_index.cpp @@ -10,6 +10,6 @@ namespace { PHLEX_REGISTER_ALGORITHMS(m, config) { m.transform("last_index", last_index, concurrency::unlimited) - .input_family("id"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "id"s})) .output_products(config.get("produces", "a")); } diff --git a/test/benchmarks/plus_101.cpp b/test/benchmarks/plus_101.cpp index c5cbd128..f70462e7 100644 --- a/test/benchmarks/plus_101.cpp +++ b/test/benchmarks/plus_101.cpp @@ -6,9 +6,9 @@ namespace { int plus_101(int i) noexcept { return i + 101; } } -PHLEX_REGISTER_ALGORITHMS(m) +PHLEX_REGISTER_ALGORITHMS(m, config) { m.transform("plus_101", plus_101, concurrency::unlimited) - .input_family("a"_in("event")) + .input_family(config.get("input")) .output_products("c"); } diff --git a/test/benchmarks/plus_one.cpp b/test/benchmarks/plus_one.cpp index 8df74f41..6caf6797 100644 --- a/test/benchmarks/plus_one.cpp +++ b/test/benchmarks/plus_one.cpp @@ -6,9 +6,9 @@ namespace { int plus_one(int i) noexcept { return i + 1; } } -PHLEX_REGISTER_ALGORITHMS(m) +PHLEX_REGISTER_ALGORITHMS(m, config) { m.transform("plus_one", plus_one, concurrency::unlimited) - .input_family("a"_in("event")) + .input_family(config.get("input")) .output_products("b"); } diff --git a/test/benchmarks/read_id.cpp b/test/benchmarks/read_id.cpp index 4ed23775..c05f9fe7 100644 --- a/test/benchmarks/read_id.cpp +++ b/test/benchmarks/read_id.cpp @@ -9,5 +9,6 @@ namespace { PHLEX_REGISTER_ALGORITHMS(m) { - m.observe("read_id", read_id, concurrency::unlimited).input_family("id"_in("event")); + m.observe("read_id", read_id, concurrency::unlimited) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "id"s})); } diff --git a/test/benchmarks/verify_difference.cpp b/test/benchmarks/verify_difference.cpp index 81fc9317..fe2452a8 100644 --- a/test/benchmarks/verify_difference.cpp +++ b/test/benchmarks/verify_difference.cpp @@ -10,6 +10,5 @@ PHLEX_REGISTER_ALGORITHMS(m, config) "verify_difference", [expected = config.get("expected", 100)](int i, int j) { assert(j - i == expected); }, concurrency::unlimited) - .input_family(product_query{config.get("i", "b"), "event"}, - product_query{config.get("j", "c"), "event"}); + .input_family(config.get("i"), config.get("j")); } diff --git a/test/cached_execution.cpp b/test/cached_execution.cpp index 6ce4f075..898b52bb 100644 --- a/test/cached_execution.cpp +++ b/test/cached_execution.cpp @@ -61,31 +61,34 @@ TEST_CASE("Cached function calls", "[data model]") // Register providers g.provide("provide_number", provide_number, concurrency::unlimited) - .output_product("number"_in("run")); + .output_product(product_query({.creator = "input"s, .layer = "run"s, .suffix = "number"s})); g.provide("provide_another", provide_another, concurrency::unlimited) - .output_product("another"_in("subrun")); + .output_product(product_query({.creator = "input"s, .layer = "subrun"s, .suffix = "another"s})); g.provide("provide_still", provide_still, concurrency::unlimited) - .output_product("still"_in("event")); + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "still"s})); g.transform("A1", call_one, concurrency::unlimited) - .input_family("number"_in("run")) + .input_family(product_query({.creator = "input"s, .layer = "run"s, .suffix = "number"s})) .output_products("one"); g.transform("A2", call_one, concurrency::unlimited) - .input_family("one"_in("run")) + .input_family(product_query({.creator = "A1"s, .layer = "run"s, .suffix = "one"s})) .output_products("used_one"); g.transform("A3", call_one, concurrency::unlimited) - .input_family("used_one"_in("run")) + .input_family(product_query({.creator = "A2"s, .layer = "run"s, .suffix = "used_one"s})) .output_products("done_one"); g.transform("B1", call_two, concurrency::unlimited) - .input_family("one"_in("run"), "another"_in("subrun")) + .input_family(product_query({.creator = "A1"s, .layer = "run"s, .suffix = "one"s}), + product_query({.creator = "input"s, .layer = "subrun"s, .suffix = "another"s})) .output_products("two"); g.transform("B2", call_two, concurrency::unlimited) - .input_family("used_one"_in("run"), "two"_in("subrun")) + .input_family(product_query({.creator = "A2"s, .layer = "run"s, .suffix = "used_one"s}), + product_query({.creator = "B1"s, .layer = "subrun"s, .suffix = "two"s})) .output_products("used_two"); g.transform("C", call_two, concurrency::unlimited) - .input_family("used_two"_in("subrun"), "still"_in("event")) + .input_family(product_query({.creator = "B2"s, .layer = "subrun"s, .suffix = "used_two"s}), + product_query({.creator = "input"s, .layer = "event"s, .suffix = "still"s})) .output_products("three"); g.execute(); diff --git a/test/class_registration.cpp b/test/class_registration.cpp index 3a7bdf6b..e640fa0f 100644 --- a/test/class_registration.cpp +++ b/test/class_registration.cpp @@ -59,17 +59,22 @@ namespace { TEST_CASE("Call non-framework functions", "[programming model]") { - std::array const product_names{"number"_in("job"), "temperature"_in("job"), "name"_in("job")}; + std::array const product_names{ + product_query({.creator = "input"s, .layer = "job"s, .suffix = "number"s}), + product_query({.creator = "input"s, .layer = "job"s, .suffix = "temperature"s}), + product_query({.creator = "input"s, .layer = "job"s, .suffix = "name"s})}; std::array const oproduct_names{"onumber"s, "otemperature"s, "oname"s}; experimental::framework_graph g{data_cell_index::base_ptr()}; // Register providers for the input products g.provide("provide_number", provide_number, concurrency::unlimited) - .output_product("number"_in("job")); + .output_product(product_query({.creator = "input"s, .layer = "job"s, .suffix = "number"s})); g.provide("provide_temperature", provide_temperature, concurrency::unlimited) - .output_product("temperature"_in("job")); - g.provide("provide_name", provide_name, concurrency::unlimited).output_product("name"_in("job")); + .output_product( + product_query({.creator = "input"s, .layer = "job"s, .suffix = "temperature"s})); + g.provide("provide_name", provide_name, concurrency::unlimited) + .output_product(product_query({.creator = "input"s, .layer = "job"s, .suffix = "name"s})); auto glueball = g.make(); SECTION("No framework") diff --git a/test/configuration.cpp b/test/configuration.cpp index 43acfa32..42c14acd 100644 --- a/test/configuration.cpp +++ b/test/configuration.cpp @@ -32,15 +32,17 @@ TEST_CASE("Retrieve value that is a configuration object", "[config]") TEST_CASE("Retrieve product_query", "[config]") { boost::json::object input; - input["product"] = "tracks"; + input["creator"] = "tracks_alg"; + input["suffix"] = "tracks"; input["layer"] = "job"; boost::json::object malformed_input1; - malformed_input1["product"] = 16.; // Should be string + malformed_input1["creator"] = "test_alg"; + malformed_input1["suffix"] = 16.; // Should be string malformed_input1["layer"] = "job"; boost::json::object malformed_input2; - malformed_input2["product"] = "hits"; + malformed_input2["creator"] = "hits"; malformed_input2["level"] = "should be layer, not level"; boost::json::object underlying_config; @@ -50,10 +52,11 @@ TEST_CASE("Retrieve product_query", "[config]") configuration config{underlying_config}; auto input_query = config.get("input"); - CHECK(input_query == "tracks"_in("job")); + CHECK( + input_query.match(product_tag{.creator = "tracks_alg"s, .layer = "job"s, .suffix = "tracks"s})); CHECK_THROWS_WITH(config.get("malformed1"), ContainsSubstring("Error retrieving parameter 'malformed1'") && - ContainsSubstring("Error retrieving parameter 'product'")); + ContainsSubstring("Error retrieving parameter 'suffix'")); CHECK_THROWS_WITH(config.get("malformed2"), ContainsSubstring("Error retrieving parameter 'malformed2'") && ContainsSubstring("Error retrieving parameter 'layer'")); diff --git a/test/demo-giantdata/unfold_transform_fold.cpp b/test/demo-giantdata/unfold_transform_fold.cpp index 77cb2eab..42599cac 100644 --- a/test/demo-giantdata/unfold_transform_fold.cpp +++ b/test/demo-giantdata/unfold_transform_fold.cpp @@ -59,7 +59,7 @@ TEST_CASE("Unfold-transform-fold pipeline", "[concurrency][unfold][fold]") spill_index.parent()->number(), spill_index.number()); }) - .output_product("wgen"_in("spill")); + .output_product(product_query({.creator = "input"s, .layer = "spill"s, .suffix = "wgen"s})); g.unfold( "WaveformGenerator", @@ -71,7 +71,7 @@ TEST_CASE("Unfold-transform-fold pipeline", "[concurrency][unfold][fold]") }, concurrency::unlimited, "APA") - .input_family("wgen"_in("spill")) + .input_family(product_query({.creator = "input"s, .layer = "spill"s, .suffix = "wgen"s})) .output_products("waves_in_apa"); // Add the transform node to the graph @@ -80,7 +80,7 @@ TEST_CASE("Unfold-transform-fold pipeline", "[concurrency][unfold][fold]") }; g.transform("clamp_node", wrapped_user_function, concurrency::unlimited) - .input_family("waves_in_apa"_in("APA")) + .input_family(product_query({.creator = "WaveformGenerator"s, .layer = "APA"s, .suffix = "waves_in_apa"s})) .output_products("clamped_waves"); // Add the fold node with instrumentation to detect pipelined execution @@ -97,7 +97,7 @@ TEST_CASE("Unfold-transform-fold pipeline", "[concurrency][unfold][fold]") }, concurrency::unlimited, "spill") - .input_family("clamped_waves"_in("APA")) + .input_family(product_query({.creator = "clamp_node"s, .layer = "APA"s, .suffix = "clamped_waves"s})) .output_products("summed_waveforms"); // Execute the graph diff --git a/test/different_hierarchies.cpp b/test/different_hierarchies.cpp index f06161d5..4c001104 100644 --- a/test/different_hierarchies.cpp +++ b/test/different_hierarchies.cpp @@ -66,22 +66,22 @@ TEST_CASE("Different hierarchies used with fold", "[graph]") // Register provider g.provide("provide_number", provide_number, concurrency::unlimited) - .output_product("number"_in("event")); + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "number"s})); g.fold("run_add", add, concurrency::unlimited, "run", 0u) - .input_family("number"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "number"s})) .output_products("run_sum"); g.fold("job_add", add, concurrency::unlimited) - .input_family("number"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "number"s})) .output_products("job_sum"); g.observe("verify_run_sum", [](unsigned int actual) { CHECK(actual == 10u); }) - .input_family("run_sum"_in("run")); + .input_family(product_query({.creator = "run_add"s, .layer = "run"s, .suffix = "run_sum"s})); g.observe("verify_job_sum", [](unsigned int actual) { CHECK(actual == 20u + 45u); // 20u from nested events, 45u from top-level events }) - .input_family("job_sum"_in("job")); + .input_family(product_query({.creator = "job_add"s, .layer = "job"s, .suffix = "job_sum"s})); g.execute(); diff --git a/test/filter.cpp b/test/filter.cpp index 4bafcf7d..490a4509 100644 --- a/test/filter.cpp +++ b/test/filter.cpp @@ -89,16 +89,19 @@ namespace { TEST_CASE("Two predicates", "[filtering]") { experimental::framework_graph g{source{10u}}; - g.provide("provide_num", give_me_nums, concurrency::unlimited).output_product("num"_in("event")); - g.predicate("evens_only", evens_only, concurrency::unlimited).input_family("num"_in("event")); - g.predicate("odds_only", odds_only, concurrency::unlimited).input_family("num"_in("event")); + g.provide("provide_num", give_me_nums, concurrency::unlimited) + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})); + g.predicate("evens_only", evens_only, concurrency::unlimited) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})); + g.predicate("odds_only", odds_only, concurrency::unlimited) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})); g.make(20u) .observe("add_evens", &sum_numbers::add, concurrency::unlimited) - .input_family("num"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})) .experimental_when("evens_only"); g.make(25u) .observe("add_odds", &sum_numbers::add, concurrency::unlimited) - .input_family("num"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})) .experimental_when("odds_only"); g.execute(); @@ -110,14 +113,16 @@ TEST_CASE("Two predicates", "[filtering]") TEST_CASE("Two predicates in series", "[filtering]") { experimental::framework_graph g{source{10u}}; - g.provide("provide_num", give_me_nums, concurrency::unlimited).output_product("num"_in("event")); - g.predicate("evens_only", evens_only, concurrency::unlimited).input_family("num"_in("event")); + g.provide("provide_num", give_me_nums, concurrency::unlimited) + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})); + g.predicate("evens_only", evens_only, concurrency::unlimited) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})); g.predicate("odds_only", odds_only, concurrency::unlimited) - .input_family("num"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})) .experimental_when("evens_only"); g.make(0u) .observe("add", &sum_numbers::add, concurrency::unlimited) - .input_family("num"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})) .experimental_when("odds_only"); g.execute(); @@ -128,12 +133,15 @@ TEST_CASE("Two predicates in series", "[filtering]") TEST_CASE("Two predicates in parallel", "[filtering]") { experimental::framework_graph g{source{10u}}; - g.provide("provide_num", give_me_nums, concurrency::unlimited).output_product("num"_in("event")); - g.predicate("evens_only", evens_only, concurrency::unlimited).input_family("num"_in("event")); - g.predicate("odds_only", odds_only, concurrency::unlimited).input_family("num"_in("event")); + g.provide("provide_num", give_me_nums, concurrency::unlimited) + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})); + g.predicate("evens_only", evens_only, concurrency::unlimited) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})); + g.predicate("odds_only", odds_only, concurrency::unlimited) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})); g.make(0u) .observe("add", &sum_numbers::add, concurrency::unlimited) - .input_family("num"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})) .experimental_when("odds_only", "evens_only"); g.execute(); @@ -153,11 +161,12 @@ TEST_CASE("Three predicates in parallel", "[filtering]") {.name = "exclude_gt_8", .begin = 8, .end = -1u}}; experimental::framework_graph g{source{10u}}; - g.provide("provide_num", give_me_nums, concurrency::unlimited).output_product("num"_in("event")); + g.provide("provide_num", give_me_nums, concurrency::unlimited) + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})); for (auto const& [name, b, e] : configs) { g.make(b, e) .predicate(name, ¬_in_range::eval, concurrency::unlimited) - .input_family("num"_in("event")); + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})); } std::vector const predicate_names{ @@ -165,7 +174,7 @@ TEST_CASE("Three predicates in parallel", "[filtering]") auto const expected_numbers = {4u, 5u, 7u}; g.make(expected_numbers) .observe("collect", &collect_numbers::collect, concurrency::unlimited) - .input_family("num"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})) .experimental_when(predicate_names); g.execute(); @@ -176,19 +185,29 @@ TEST_CASE("Three predicates in parallel", "[filtering]") TEST_CASE("Two predicates in parallel (each with multiple arguments)", "[filtering]") { experimental::framework_graph g{source{10u}}; - g.provide("provide_num", give_me_nums, concurrency::unlimited).output_product("num"_in("event")); + g.provide("provide_num", give_me_nums, concurrency::unlimited) + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})); g.provide("provide_other_num", give_me_other_nums, concurrency::unlimited) - .output_product("other_num"_in("event")); - g.predicate("evens_only", evens_only, concurrency::unlimited).input_family("num"_in("event")); - g.predicate("odds_only", odds_only, concurrency::unlimited).input_family("num"_in("event")); + .output_product( + product_query({.creator = "input"s, .layer = "event"s, .suffix = "other_num"s})); + g.predicate("evens_only", evens_only, concurrency::unlimited) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})); + g.predicate("odds_only", odds_only, concurrency::unlimited) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s})); g.make(5 * 100) .observe("check_evens", &check_multiple_numbers::add_difference, concurrency::unlimited) - .input_family("num"_in("event"), "other_num"_in("event")) // <= Note input order + .input_family( + product_query({.creator = "input"s, .layer = "event"s, .suffix = "num"s}), + product_query( + {.creator = "input"s, .layer = "event"s, .suffix = "other_num"s})) // <= Note input order .experimental_when("evens_only"); g.make(-5 * 100) .observe("check_odds", &check_multiple_numbers::add_difference, concurrency::unlimited) - .input_family("other_num"_in("event"), "num"_in("event")) // <= Note input order + .input_family( + product_query({.creator = "input"s, .layer = "event"s, .suffix = "other_num"s}), + product_query( + {.creator = "input"s, .layer = "event"s, .suffix = "num"s})) // <= Note input order .experimental_when("odds_only"); g.execute(); diff --git a/test/filter_impl.cpp b/test/filter_impl.cpp index 27920b38..3cc7e728 100644 --- a/test/filter_impl.cpp +++ b/test/filter_impl.cpp @@ -45,8 +45,10 @@ TEST_CASE("Filter decision", "[filtering]") TEST_CASE("Filter data map", "[filtering]") { - using phlex::operator""_in; - std::vector const data_products_to_cache{"a"_in("spill"), "b"_in("spill")}; + using phlex::product_query; + std::vector const data_products_to_cache{ + product_query({.creator = "input"s, .layer = "spill"s, .suffix = "a"s}), + product_query({.creator = "input"s, .layer = "spill"s, .suffix = "b"s})}; data_map data{data_products_to_cache}; // Stores with the data products "a" and "b" diff --git a/test/fold.cpp b/test/fold.cpp index 62fa4b3b..5dd4a626 100644 --- a/test/fold.cpp +++ b/test/fold.cpp @@ -53,25 +53,26 @@ TEST_CASE("Different data layers of fold", "[graph]") experimental::framework_graph g{driver_for_test(gen)}; g.provide("provide_number", provide_number, concurrency::unlimited) - .output_product("number"_in("event")); + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "number"s})); g.fold("run_add", add, concurrency::unlimited, "run") - .input_family("number"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "number"s})) .output_products("run_sum"); g.fold("job_add", add, concurrency::unlimited) - .input_family("number"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "number"s})) .output_products("job_sum"); g.fold("two_layer_job_add", add, concurrency::unlimited) - .input_family("run_sum"_in("run")) + .input_family(product_query({.creator = "run_add"s, .layer = "run"s, .suffix = "run_sum"s})) .output_products("two_layer_job_sum"); g.observe("verify_run_sum", [](unsigned int actual) { CHECK(actual == 10u); }) - .input_family("run_sum"_in("run")); + .input_family(product_query({.creator = "run_add"s, .layer = "run"s, .suffix = "run_sum"s})); g.observe("verify_two_layer_job_sum", [](unsigned int actual) { CHECK(actual == 20u); }) - .input_family("two_layer_job_sum"_in("job")); + .input_family(product_query( + {.creator = "two_layer_job_add"s, .layer = "job"s, .suffix = "two_layer_job_sum"s})); g.observe("verify_job_sum", [](unsigned int actual) { CHECK(actual == 20u); }) - .input_family("job_sum"_in("job")); + .input_family(product_query({.creator = "job_add"s, .layer = "job"s, .suffix = "job_sum"s})); g.execute(); diff --git a/test/framework_graph.cpp b/test/framework_graph.cpp index beb879ee..1ff09d3b 100644 --- a/test/framework_graph.cpp +++ b/test/framework_graph.cpp @@ -29,10 +29,10 @@ TEST_CASE("Make progress with one thread", "[graph]") "provide_number", [](data_cell_index const& index) -> unsigned int { return index.number(); }, concurrency::unlimited) - .output_product("number"_in("spill")); + .output_product(product_query({.creator = "input"s, .layer = "spill"s, .suffix = "number"s})); g.observe( "observe_number", [](unsigned int const /*number*/) {}, concurrency::unlimited) - .input_family("number"_in("spill")); + .input_family(product_query({.creator = "input"s, .layer = "spill"s, .suffix = "number"s})); g.execute(); CHECK(gen.emitted_cells("/job/spill") == 1000); diff --git a/test/function_registration.cpp b/test/function_registration.cpp index 552560d5..e7a1de06 100644 --- a/test/function_registration.cpp +++ b/test/function_registration.cpp @@ -54,7 +54,10 @@ namespace { TEST_CASE("Call non-framework functions", "[programming model]") { - std::array const product_names{"number"_in("job"), "temperature"_in("job"), "name"_in("job")}; + std::array const product_names{ + product_query({.creator = "input"s, .layer = "job"s, .suffix = "number"s}), + product_query({.creator = "input"s, .layer = "job"s, .suffix = "temperature"s}), + product_query({.creator = "input"s, .layer = "job"s, .suffix = "name"s})}; std::array const oproduct_names = {"onumber"s, "otemperature"s, "oname"s}; std::array const result{"result"s}; @@ -62,10 +65,12 @@ TEST_CASE("Call non-framework functions", "[programming model]") // Register providers g.provide("provide_number", provide_number, concurrency::unlimited) - .output_product("number"_in("job")); + .output_product(product_query({.creator = "input"s, .layer = "job"s, .suffix = "number"s})); g.provide("provide_temperature", provide_temperature, concurrency::unlimited) - .output_product("temperature"_in("job")); - g.provide("provide_name", provide_name, concurrency::unlimited).output_product("name"_in("job")); + .output_product( + product_query({.creator = "input"s, .layer = "job"s, .suffix = "temperature"s})); + g.provide("provide_name", provide_name, concurrency::unlimited) + .output_product(product_query({.creator = "input"s, .layer = "job"s, .suffix = "name"s})); SECTION("No framework") { diff --git a/test/hierarchical_nodes.cpp b/test/hierarchical_nodes.cpp index 457aa4e8..811487fc 100644 --- a/test/hierarchical_nodes.cpp +++ b/test/hierarchical_nodes.cpp @@ -93,7 +93,7 @@ TEST_CASE("Hierarchical nodes", "[graph]") spdlog::info("Providing time for {}", index.to_string()); return std::time(nullptr); }) - .output_product("time"_in("run")); + .output_product(product_query({.creator = "input"s, .layer = "run"s, .suffix = "time"s})); g.provide("provide_number", [](data_cell_index const& index) -> unsigned int { @@ -101,26 +101,29 @@ TEST_CASE("Hierarchical nodes", "[graph]") auto const run_number = index.parent()->number(); return event_number + run_number; }) - .output_product("number"_in("event")); + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "number"s})); g.transform("get_the_time", strtime, concurrency::unlimited) - .input_family("time"_in("run")) + .input_family(product_query({.creator = "input"s, .layer = "run"s, .suffix = "time"s})) .experimental_when() .output_products("strtime"); g.transform("square", square, concurrency::unlimited) - .input_family("number"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "number"s})) .output_products("squared_number"); g.fold("add", add, concurrency::unlimited, "run", 15u) - .input_family("squared_number"_in("event")) + .input_family( + product_query({.creator = "square"s, .layer = "event"s, .suffix = "squared_number"s})) .experimental_when() .output_products("added_data"); g.transform("scale", scale, concurrency::unlimited) - .input_family("added_data"_in("run")) + .input_family(product_query({.creator = "add"s, .layer = "run"s, .suffix = "added_data"s})) .output_products("result"); g.observe("print_result", print_result, concurrency::unlimited) - .input_family("result"_in("run"), "strtime"_in("run")); + .input_family( + product_query({.creator = "scale"s, .layer = "run"s, .suffix = "result"s}), + product_query({.creator = "get_the_time"s, .layer = "run"s, .suffix = "strtime"s})); g.make() .output("save", &experimental::test::products_for_output::save) diff --git a/test/max-parallelism/check_parallelism.cpp b/test/max-parallelism/check_parallelism.cpp index 26cd719b..8c18b244 100644 --- a/test/max-parallelism/check_parallelism.cpp +++ b/test/max-parallelism/check_parallelism.cpp @@ -15,5 +15,6 @@ PHLEX_REGISTER_ALGORITHMS(m, config) [expected = config.get("expected_parallelism")](std::size_t actual) { assert(actual == expected); }) - .input_family("max_parallelism"_in("job")); + .input_family( + product_query({.creator = "input"s, .layer = "job"s, .suffix = "max_parallelism"s})); } diff --git a/test/max-parallelism/provide_parallelism.cpp b/test/max-parallelism/provide_parallelism.cpp index c41f1381..e763e8bf 100644 --- a/test/max-parallelism/provide_parallelism.cpp +++ b/test/max-parallelism/provide_parallelism.cpp @@ -7,5 +7,6 @@ PHLEX_REGISTER_PROVIDERS(s) s.provide( "provide_max_parallelism", [](data_cell_index const&) { return experimental::max_allowed_parallelism::active_value(); }) - .output_product("max_parallelism"_in("job")); + .output_product( + product_query({.creator = "input"s, .layer = "job"s, .suffix = "max_parallelism"s})); } diff --git a/test/memory-checks/many_events.cpp b/test/memory-checks/many_events.cpp index c6e4fff4..eaf7bb1b 100644 --- a/test/memory-checks/many_events.cpp +++ b/test/memory-checks/many_events.cpp @@ -19,9 +19,9 @@ int main() experimental::framework_graph g{driver_for_test(gen)}; g.provide("provide_number", [](data_cell_index const& id) -> unsigned { return id.number(); }) - .output_product("number"_in("event")); + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "number"s})); g.transform("pass_on", pass_on, concurrency::unlimited) - .input_family("number"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "number"s})) .output_products("different"); g.execute(); } diff --git a/test/mock-workflow/G4Stage1.libsonnet b/test/mock-workflow/G4Stage1.libsonnet index df22de14..c5f1eae7 100644 --- a/test/mock-workflow/G4Stage1.libsonnet +++ b/test/mock-workflow/G4Stage1.libsonnet @@ -5,7 +5,7 @@ local generators = import 'SinglesGen.libsonnet'; largeant: { cpp: 'largeant', duration_usec: 156, # Typical: 15662051 - inputs: [ev.event_product(f + "/MCTruths") for f in std.objectFields(generators)], + inputs: [ev.creator_event_product(f, "MCTruths") for f in std.objectFields(generators)], outputs: ["ParticleAncestryMap", "Assns", "SimEnergyDeposits", "AuxDetHits", "MCParticles"], } } diff --git a/test/mock-workflow/G4Stage2.libsonnet b/test/mock-workflow/G4Stage2.libsonnet index 535e90a7..dae76be9 100644 --- a/test/mock-workflow/G4Stage2.libsonnet +++ b/test/mock-workflow/G4Stage2.libsonnet @@ -5,13 +5,13 @@ local g4stage1 = import 'G4Stage1.libsonnet'; IonAndScint: { cpp: 'ion_and_scint', duration_usec: 546, # Typical: 5457973 - inputs: [ev.event_product(f + "/SimEnergyDeposits") for f in std.objectFields(g4stage1)], + inputs: [ev.creator_event_product(f, "SimEnergyDeposits") for f in std.objectFields(g4stage1)], outputs: ["SimEnergyDeposits", "SimEnergyDeposits_priorSCE"], }, PDFastSim: { cpp: 'pd_fast_sim', duration_usec: 69, # Typical: 69681950 - inputs: [ev.event_product('SimEnergyDeposits_priorSCE')], + inputs: [ev.creator_event_product('IonAndScint', 'SimEnergyDeposits_priorSCE')], outputs: ['SimPhotonLites', 'OpDetBacktrackerRecords'], } } diff --git a/test/mock-workflow/event_product.libsonnet b/test/mock-workflow/event_product.libsonnet index a3f0cfc4..6f3bb2ac 100644 --- a/test/mock-workflow/event_product.libsonnet +++ b/test/mock-workflow/event_product.libsonnet @@ -1,7 +1,15 @@ { event_product(product):: { - product: product, + creator: 'input', + suffix: product, + layer: "event" + }, + + creator_event_product(creator, product):: + { + creator: creator, + suffix: product, layer: "event" } } diff --git a/test/mock-workflow/id_provider.cpp b/test/mock-workflow/id_provider.cpp index 126bd676..b0dba07b 100644 --- a/test/mock-workflow/id_provider.cpp +++ b/test/mock-workflow/id_provider.cpp @@ -4,5 +4,5 @@ PHLEX_REGISTER_PROVIDERS(s) { using namespace phlex; s.provide("provide_id", [](data_cell_index const& id) { return id; }) - .output_product("id"_in("event")); + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "id"s})); } diff --git a/test/multiple_function_registration.cpp b/test/multiple_function_registration.cpp index fa338c70..5d74c664 100644 --- a/test/multiple_function_registration.cpp +++ b/test/multiple_function_registration.cpp @@ -45,41 +45,47 @@ TEST_CASE("Call multiple functions", "[programming model]") g.provide("provide_numbers", [](data_cell_index const&) -> std::vector { return {0, 1, 2, 3, 4}; }) - .output_product("numbers"_in("job")); + .output_product(product_query({.creator = "input"s, .layer = "job"s, .suffix = "numbers"s})); g.provide("provide_offset", [](data_cell_index const&) -> unsigned { return 6u; }) - .output_product("offset"_in("job")); + .output_product(product_query({.creator = "input"s, .layer = "job"s, .suffix = "offset"s})); SECTION("All free functions") { g.transform("square_numbers", square_numbers, concurrency::unlimited) - .input_family("numbers"_in("job")) + .input_family(product_query({.creator = "input"s, .layer = "job"s, .suffix = "numbers"s})) .output_products("squared_numbers"); g.transform("sum_numbers", sum_numbers, concurrency::unlimited) - .input_family("squared_numbers"_in("job")) + .input_family(product_query( + {.creator = "square_numbers"s, .layer = "job"s, .suffix = "squared_numbers"s})) .output_products("summed_numbers"); - g.transform("sqrt_sum_numbers", sqrt_sum_numbers, concurrency::unlimited) - .input_family("summed_numbers"_in("job"), "offset"_in("job")) + g.transform("sqrt_sum", sqrt_sum_numbers, concurrency::unlimited) + .input_family( + product_query({.creator = "sum_numbers"s, .layer = "job"s, .suffix = "summed_numbers"s}), + product_query({.creator = "input"s, .layer = "job"s, .suffix = "offset"s})) .output_products("result"); } SECTION("Transforms, one from a class") { g.transform("square_numbers", square_numbers, concurrency::unlimited) - .input_family("numbers"_in("job")) + .input_family(product_query({.creator = "input"s, .layer = "job"s, .suffix = "numbers"s})) .output_products("squared_numbers"); g.transform("sum_numbers", sum_numbers, concurrency::unlimited) - .input_family("squared_numbers"_in("job")) + .input_family(product_query( + {.creator = "square_numbers"s, .layer = "job"s, .suffix = "squared_numbers"s})) .output_products("summed_numbers"); g.make() .transform("sqrt_sum", &A::sqrt_sum, concurrency::unlimited) - .input_family("summed_numbers"_in("job"), "offset"_in("job")) + .input_family( + product_query({.creator = "sum_numbers"s, .layer = "job"s, .suffix = "summed_numbers"s}), + product_query({.creator = "input"s, .layer = "job"s, .suffix = "offset"s})) .output_products("result"); } // The following is invoked for *each* section above g.observe("verify_result", [](double actual) { assert(actual == 6.); }) - .input_family("result"_in("job")); + .input_family(product_query({.creator = "sqrt_sum"s, .layer = "job"s, .suffix = "result"s})); g.execute(); } diff --git a/test/plugins/ij_source.cpp b/test/plugins/ij_source.cpp index 0eec1c18..89a05d15 100644 --- a/test/plugins/ij_source.cpp +++ b/test/plugins/ij_source.cpp @@ -5,7 +5,7 @@ using namespace phlex; PHLEX_REGISTER_PROVIDERS(s) { s.provide("provide_i", [](data_cell_index const& id) -> int { return id.number(); }) - .output_product("i"_in("event")); + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "i"s})); s.provide("provide_j", [](data_cell_index const& id) -> int { return -id.number(); }) - .output_product("j"_in("event")); + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "j"s})); } diff --git a/test/plugins/module.cpp b/test/plugins/module.cpp index db189077..9194e3f9 100644 --- a/test/plugins/module.cpp +++ b/test/plugins/module.cpp @@ -8,9 +8,10 @@ using namespace phlex; PHLEX_REGISTER_ALGORITHMS(m) { m.transform("add", test::add, concurrency::unlimited) - .input_family("i"_in("event"), "j"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "i"s}), + product_query({.creator = "input"s, .layer = "event"s, .suffix = "j"s})) .output_products("sum"); m.observe( "verify", [](int actual) { assert(actual == 0); }, concurrency::unlimited) - .input_family("sum"_in("event")); + .input_family(product_query({.creator = "add"s, .layer = "event"s, .suffix = "sum"s})); } diff --git a/test/product_query.cpp b/test/product_query.cpp index ade872d6..68faa36d 100644 --- a/test/product_query.cpp +++ b/test/product_query.cpp @@ -7,15 +7,18 @@ using namespace phlex; TEST_CASE("Empty specifications", "[data model]") { - CHECK_THROWS_WITH(""_in, - Catch::Matchers::ContainsSubstring("Cannot specify product with empty name.")); CHECK_THROWS_WITH( - "product"_in(""), + product_query({.creator = ""s, .layer = "layer"s}), + Catch::Matchers::ContainsSubstring("Cannot specify product with empty creator name.")); + CHECK_THROWS_WITH( + product_query({.creator = "creator"s, .layer = ""s}), Catch::Matchers::ContainsSubstring("Cannot specify the empty string as a data layer.")); } TEST_CASE("Product name with data layer", "[data model]") { - product_query label{"product", {"event"}}; - CHECK(label == "product"_in("event")); + product_query label({.creator = "creator"s, .layer = "event"s, .suffix = "product"s}); + CHECK(label.creator() == "creator"s); + CHECK(label.layer() == "event"s); + CHECK(label.suffix() == "product"s); } diff --git a/test/provider_test.cpp b/test/provider_test.cpp index d8e6281d..26d69270 100644 --- a/test/provider_test.cpp +++ b/test/provider_test.cpp @@ -39,10 +39,12 @@ TEST_CASE("provider_test") experimental::framework_graph g{driver_for_test(gen)}; g.provide("my_name_here", give_me_vertices, concurrency::unlimited) - .output_product("happy_vertices"_in("spill")); + .output_product( + product_query({.creator = "input"s, .layer = "spill"s, .suffix = "happy_vertices"s})); g.transform("passer", pass_on, concurrency::unlimited) - .input_family("happy_vertices"_in("spill")) + .input_family( + product_query({.creator = "input"s, .layer = "spill"s, .suffix = "happy_vertices"s})) .output_products("vertex_data"); g.execute(); diff --git a/test/python/source.cpp b/test/python/source.cpp index 2a6aac8f..0145eb94 100644 --- a/test/python/source.cpp +++ b/test/python/source.cpp @@ -6,7 +6,7 @@ using namespace phlex; PHLEX_REGISTER_PROVIDERS(s) { s.provide("provide_i", [](data_cell_index const& id) -> int { return id.number(); }) - .output_product("i"_in("job")); + .output_product(product_query({.creator = "input"s, .layer = "job"s, .suffix = "i"s})); s.provide("provide_j", [](data_cell_index const& id) -> int { return -id.number() + 1; }) - .output_product("j"_in("job")); + .output_product(product_query({.creator = "input"s, .layer = "job"s, .suffix = "j"s})); } diff --git a/test/type_distinction.cpp b/test/type_distinction.cpp index c9a71ef3..0cb18af8 100644 --- a/test/type_distinction.cpp +++ b/test/type_distinction.cpp @@ -57,55 +57,65 @@ TEST_CASE("Distinguish products with same name and different types", "[programmi // Register providers g.provide("provide_numbers", provide_numbers, concurrency::unlimited) - .output_product("numbers"_in("event")); + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "numbers"s})); g.provide("provide_length", provide_length, concurrency::unlimited) - .output_product("length"_in("event")); + .output_product(product_query({.creator = "input"s, .layer = "event"s, .suffix = "length"s})); - SECTION("Duplicate product name but differ in producer name") + SECTION("Duplicate product name but differ in creator name") { g.observe("starter", [](int num) { spdlog::info("Received {}", num); }) - .input_family("numbers"_in("event")); + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "numbers"s})); g.transform("triple_numbers", triple, concurrency::unlimited) - .input_family("numbers"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "numbers"s})) .output_products("tripled"); spdlog::info("Registered tripled"); g.transform("expand_orig", expand, concurrency::unlimited) - .input_family("numbers"_in("event"), "length"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "numbers"s}), + product_query({.creator = "input"s, .layer = "event"s, .suffix = "length"s})) .output_products("expanded_one"); spdlog::info("Registered expanded_one"); g.transform("expand_triples", expand, concurrency::unlimited) - .input_family("tripled"_in("event"), "length"_in("event")) + .input_family( + product_query({.creator = "triple_numbers"s, .layer = "event"s, .suffix = "tripled"s}), + product_query({.creator = "input"s, .layer = "event"s, .suffix = "length"s})) .output_products("expanded_three"); spdlog::info("Registered expanded_three"); g.transform("add_nums", add_numbers, concurrency::unlimited) - .input_family("numbers"_in("event"), "tripled"_in("event")) + .input_family( + product_query({.creator = "input"s, .layer = "event"s, .suffix = "numbers"s}), + product_query({.creator = "triple_numbers"s, .layer = "event"s, .suffix = "tripled"s})) .output_products("sums"); spdlog::info("Registered sums"); g.transform("add_vect", add_vectors, concurrency::unlimited) - .input_family("expanded_one"_in("event"), "expanded_three"_in("event")) + .input_family( + product_query({.creator = "expand_orig"s, .layer = "event"s, .suffix = "expanded_one"s}), + product_query( + {.creator = "expand_triples"s, .layer = "event"s, .suffix = "expanded_three"s})) .output_products("sums"); - g.transform("test_add_num", triple, concurrency::unlimited) - .input_family("sums"_in("event")) + g.transform("extract_result", triple, concurrency::unlimited) + .input_family(product_query({.creator = "add_nums"s, .layer = "event"s, .suffix = "sums"s})) .output_products("result"); spdlog::info("Registered result"); } - SECTION("Duplicate product name and producer, differ only in type") + SECTION("Duplicate product name and creator, differ only in type") { g.transform("square", square, concurrency::unlimited) - .input_family("numbers"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "numbers"s})) .output_products("square_result", "square_result"); g.transform("extract_result", id, concurrency::unlimited) - .input_family("square_result"_in("event")) + .input_family( + product_query({.creator = "square"s, .layer = "event"s, .suffix = "square_result"s})) .output_products("result"); } g.observe("print_result", [](int res) { spdlog::info("Result: {}", res); }) - .input_family("result"_in("event")); + .input_family( + product_query({.creator = "extract_result"s, .layer = "event"s, .suffix = "result"s})); spdlog::info("Registered observe"); g.execute(); spdlog::info("Executed"); diff --git a/test/unfold.cpp b/test/unfold.cpp index faee73fd..1652ca95 100644 --- a/test/unfold.cpp +++ b/test/unfold.cpp @@ -97,30 +97,34 @@ TEST_CASE("Splitting the processing", "[graph]") experimental::framework_graph g{driver_for_test(gen)}; g.provide("provide_max_number", provide_max_number, concurrency::unlimited) - .output_product("max_number"_in("event")); + .output_product( + product_query({.creator = "input"s, .layer = "event"s, .suffix = "max_number"s})); g.provide("provide_ten_numbers", provide_ten_numbers, concurrency::unlimited) - .output_product("ten_numbers"_in("event")); + .output_product( + product_query({.creator = "input"s, .layer = "event"s, .suffix = "ten_numbers"s})); g.unfold("iota", &iota::predicate, &iota::unfold, concurrency::unlimited, "lower1") - .input_family("max_number"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "max_number"s})) .output_products("new_number"); g.fold("add", add, concurrency::unlimited, "event") - .input_family("new_number"_in("lower1")) + .input_family(product_query({.creator = "iota"s, .layer = "lower1"s, .suffix = "new_number"s})) .output_products("sum1"); - g.observe("check_sum", check_sum, concurrency::unlimited).input_family("sum1"_in("event")); + g.observe("check_sum", check_sum, concurrency::unlimited) + .input_family(product_query({.creator = "add"s, .layer = "event"s, .suffix = "sum1"s})); g.unfold("iterate_through", &iterate_through::predicate, &iterate_through::unfold, concurrency::unlimited, "lower2") - .input_family("ten_numbers"_in("event")) + .input_family(product_query({.creator = "input"s, .layer = "event"s, .suffix = "ten_numbers"s})) .output_products("each_number"); g.fold("add_numbers", add_numbers, concurrency::unlimited, "event") - .input_family("each_number"_in("lower2")) + .input_family( + product_query({.creator = "iterate_through"s, .layer = "lower2"s, .suffix = "each_number"s})) .output_products("sum2"); g.observe("check_sum_same", check_sum_same, concurrency::unlimited) - .input_family("sum2"_in("event")); + .input_family(product_query({.creator = "add_numbers"s, .layer = "event"s, .suffix = "sum2"s})); g.make().output( "save", &experimental::test::products_for_output::save, concurrency::serial); diff --git a/test/vector_of_abstract_types.cpp b/test/vector_of_abstract_types.cpp index 6da3234d..80feab7e 100644 --- a/test/vector_of_abstract_types.cpp +++ b/test/vector_of_abstract_types.cpp @@ -44,11 +44,13 @@ TEST_CASE("Test vector of abstract types") experimental::framework_graph g{driver_for_test(gen)}; g.provide("provide_thing", [](data_cell_index const&) { return make_derived_as_abstract(); }) - .output_product("thing"_in("event")); - g.transform("read_thing", read_abstract).input_family("thing"_in("event")).output_products("sum"); + .output_product(product_query({.creator = "dummy"s, .layer = "event"s, .suffix = "thing"s})); + g.transform("read_thing", read_abstract) + .input_family(product_query({.creator = "dummy"s, .layer = "event"s, .suffix = "thing"s})) + .output_products("sum"); g.observe( "verify_sum", [](int sum) { CHECK(sum == 3); }, concurrency::serial) - .input_family("sum"_in("event")); + .input_family(product_query({.creator = "read_thing"s, .layer = "event"s, .suffix = "sum"s})); g.execute(); CHECK(g.execution_counts("read_thing") == 1); From e8b9e258352b494aeb8a0c1776156ab1df0a7c00 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 26 Jan 2026 19:01:07 +0000 Subject: [PATCH 3/7] Apply clang-format fixes --- phlex/configuration.cpp | 6 ++++-- phlex/core/edge_creation_policy.cpp | 6 +++--- phlex/model/type_id.hpp | 7 +++++-- test/demo-giantdata/unfold_transform_fold.cpp | 6 ++++-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/phlex/configuration.cpp b/phlex/configuration.cpp index b120ffc7..dab4f911 100644 --- a/phlex/configuration.cpp +++ b/phlex/configuration.cpp @@ -42,8 +42,10 @@ namespace { kind = "HOW??"s; break; } - throw std::runtime_error(fmt::format( - "Error retrieving parameter '{}'. Should be a string but is instead a {}", parameter, kind)); + throw std::runtime_error( + fmt::format("Error retrieving parameter '{}'. Should be a string but is instead a {}", + parameter, + kind)); } return std::string(val.get_string()); } diff --git a/phlex/core/edge_creation_policy.cpp b/phlex/core/edge_creation_policy.cpp index 22924b6b..5a3de719 100644 --- a/phlex/core/edge_creation_policy.cpp +++ b/phlex/core/edge_creation_policy.cpp @@ -44,9 +44,9 @@ namespace phlex::experimental { } candidates.emplace(producer.node.full(), &producer); } - } - else { - spdlog::error("Creator name mismatch between ({}) and {}", query.to_string(), producer.node.full()); + } else { + spdlog::error( + "Creator name mismatch between ({}) and {}", query.to_string(), producer.node.full()); } } diff --git a/phlex/model/type_id.hpp b/phlex/model/type_id.hpp index d7ce943e..080f3e88 100644 --- a/phlex/model/type_id.hpp +++ b/phlex/model/type_id.hpp @@ -45,8 +45,11 @@ namespace phlex::experimental { constexpr builtin fundamental() const { return static_cast(id_ & 0x0F); } template - friend constexpr void tag_invoke( - boost::hash2::hash_append_tag const&, Provider const&, Hash& h, Flavor const& f, type_id const* v) + friend constexpr void tag_invoke(boost::hash2::hash_append_tag const&, + Provider const&, + Hash& h, + Flavor const& f, + type_id const* v) { boost::hash2::hash_append(h, f, v->id_); if (v->has_children()) { diff --git a/test/demo-giantdata/unfold_transform_fold.cpp b/test/demo-giantdata/unfold_transform_fold.cpp index 42599cac..c0678553 100644 --- a/test/demo-giantdata/unfold_transform_fold.cpp +++ b/test/demo-giantdata/unfold_transform_fold.cpp @@ -80,7 +80,8 @@ TEST_CASE("Unfold-transform-fold pipeline", "[concurrency][unfold][fold]") }; g.transform("clamp_node", wrapped_user_function, concurrency::unlimited) - .input_family(product_query({.creator = "WaveformGenerator"s, .layer = "APA"s, .suffix = "waves_in_apa"s})) + .input_family( + product_query({.creator = "WaveformGenerator"s, .layer = "APA"s, .suffix = "waves_in_apa"s})) .output_products("clamped_waves"); // Add the fold node with instrumentation to detect pipelined execution @@ -97,7 +98,8 @@ TEST_CASE("Unfold-transform-fold pipeline", "[concurrency][unfold][fold]") }, concurrency::unlimited, "spill") - .input_family(product_query({.creator = "clamp_node"s, .layer = "APA"s, .suffix = "clamped_waves"s})) + .input_family( + product_query({.creator = "clamp_node"s, .layer = "APA"s, .suffix = "clamped_waves"s})) .output_products("summed_waveforms"); // Execute the graph From 33567e42fa4cb9ae36d40fb2c29f483c82a54d51 Mon Sep 17 00:00:00 2001 From: Beojan Stanislaus Date: Mon, 26 Jan 2026 11:50:45 -0800 Subject: [PATCH 4/7] Format Jsonnet files --- test/benchmarks/benchmark-04.jsonnet | 44 ++++++------ test/benchmarks/benchmark-05.jsonnet | 30 ++++----- test/benchmarks/benchmark-06.jsonnet | 70 +++++++++---------- test/benchmarks/benchmark-07.jsonnet | 78 +++++++++++----------- test/benchmarks/benchmark-08.jsonnet | 14 ++-- test/benchmarks/benchmark-09.jsonnet | 16 ++--- test/mock-workflow/G4Stage1.libsonnet | 10 +-- test/mock-workflow/G4Stage2.libsonnet | 12 ++-- test/mock-workflow/event_product.libsonnet | 10 +-- 9 files changed, 142 insertions(+), 142 deletions(-) diff --git a/test/benchmarks/benchmark-04.jsonnet b/test/benchmarks/benchmark-04.jsonnet index 18f5287d..49f4385e 100644 --- a/test/benchmarks/benchmark-04.jsonnet +++ b/test/benchmarks/benchmark-04.jsonnet @@ -1,26 +1,26 @@ { - "driver": { - "cpp": "generate_layers", - "layers": { - "event": { "total": 100000 } - } + driver: { + cpp: 'generate_layers', + layers: { + event: { total: 100000 }, }, - "sources": { - "provider": { - "cpp": "benchmarks_provider" - } + }, + sources: { + provider: { + cpp: 'benchmarks_provider', }, - "modules": { - "a_creator": { - "cpp": "last_index" - }, - "read_index": { - "cpp": "read_index", - "consumes": { - "creator": "a_creator", - "suffix": "a", - "layer": "event" - } - } - } + }, + modules: { + a_creator: { + cpp: 'last_index', + }, + read_index: { + cpp: 'read_index', + consumes: { + creator: 'a_creator', + suffix: 'a', + layer: 'event', + }, + }, + }, } diff --git a/test/benchmarks/benchmark-05.jsonnet b/test/benchmarks/benchmark-05.jsonnet index 9559ea5b..9f5a7708 100644 --- a/test/benchmarks/benchmark-05.jsonnet +++ b/test/benchmarks/benchmark-05.jsonnet @@ -2,13 +2,13 @@ driver: { cpp: 'generate_layers', layers: { - event: { total: 100000 } - } + event: { total: 100000 }, + }, }, sources: { provider: { - cpp: 'benchmarks_provider' - } + cpp: 'benchmarks_provider', + }, }, modules: { b_creator: { @@ -21,17 +21,17 @@ }, d: { cpp: 'verify_difference', - i: { - creator: 'b_creator', - layer: 'event', - suffix: 'b' - }, - j: { - creator: 'c_creator', - layer: 'event', - suffix: 'c' - }, - expected: 0 + i: { + creator: 'b_creator', + layer: 'event', + suffix: 'b', + }, + j: { + creator: 'c_creator', + layer: 'event', + suffix: 'c', + }, + expected: 0, }, }, } diff --git a/test/benchmarks/benchmark-06.jsonnet b/test/benchmarks/benchmark-06.jsonnet index 879d9c1b..45680092 100644 --- a/test/benchmarks/benchmark-06.jsonnet +++ b/test/benchmarks/benchmark-06.jsonnet @@ -1,39 +1,39 @@ { - "driver": { - "cpp": "generate_layers", - "layers": { - "event": { "total": 100000 } - } + driver: { + cpp: 'generate_layers', + layers: { + event: { total: 100000 }, }, - "sources": { - "provider": { - "cpp": "benchmarks_provider" - } + }, + sources: { + provider: { + cpp: 'benchmarks_provider', }, - "modules": { - "a_creator": { - "cpp": "last_index" - }, - "b_creator": { - "cpp": "plus_one", - "input": { "creator": "a_creator", "layer": "event", "suffix": "a" } - }, - "c_creator": { - "cpp": "plus_101", - "input": { "creator": "a_creator", "layer": "event", "suffix": "a" } - }, - "d": { - "cpp": "verify_difference", - "i": { - "creator": "b_creator", - "layer": "event", - "suffix": "b" - }, - "j": { - "creator": "c_creator", - "layer": "event", - "suffix": "c" - } - } - } + }, + modules: { + a_creator: { + cpp: 'last_index', + }, + b_creator: { + cpp: 'plus_one', + input: { creator: 'a_creator', layer: 'event', suffix: 'a' }, + }, + c_creator: { + cpp: 'plus_101', + input: { creator: 'a_creator', layer: 'event', suffix: 'a' }, + }, + d: { + cpp: 'verify_difference', + i: { + creator: 'b_creator', + layer: 'event', + suffix: 'b', + }, + j: { + creator: 'c_creator', + layer: 'event', + suffix: 'c', + }, + }, + }, } diff --git a/test/benchmarks/benchmark-07.jsonnet b/test/benchmarks/benchmark-07.jsonnet index 026f9b72..b7564d37 100644 --- a/test/benchmarks/benchmark-07.jsonnet +++ b/test/benchmarks/benchmark-07.jsonnet @@ -1,43 +1,43 @@ { - "driver": { - "cpp": "generate_layers", - "layers": { - "event": { "total": 100000 } - } + driver: { + cpp: 'generate_layers', + layers: { + event: { total: 100000 }, }, - "sources": { - "provider": { - "cpp": "benchmarks_provider" - } + }, + sources: { + provider: { + cpp: 'benchmarks_provider', }, - "modules": { - "even_filter": { - "cpp": "accept_even_ids", - "input": { "creator": "input", "suffix": "id", "layer": "event" } - }, - "b_creator": { - "cpp": "last_index", - "experimental_when": ["even_filter:accept_even_ids"], - "produces": "b" - }, - "c_creator": { - "cpp": "last_index", - "experimental_when": ["even_filter:accept_even_ids"], - "produces": "c" - }, - "d": { - "cpp": "verify_difference", - "i": { - "creator": "b_creator", - "layer": "event", - "suffix": "b" - }, - "j": { - "creator": "c_creator", - "layer": "event", - "suffix": "c" - }, - "expected": 0 - } - } + }, + modules: { + even_filter: { + cpp: 'accept_even_ids', + input: { creator: 'input', suffix: 'id', layer: 'event' }, + }, + b_creator: { + cpp: 'last_index', + experimental_when: ['even_filter:accept_even_ids'], + produces: 'b', + }, + c_creator: { + cpp: 'last_index', + experimental_when: ['even_filter:accept_even_ids'], + produces: 'c', + }, + d: { + cpp: 'verify_difference', + i: { + creator: 'b_creator', + layer: 'event', + suffix: 'b', + }, + j: { + creator: 'c_creator', + layer: 'event', + suffix: 'c', + }, + expected: 0, + }, + }, } diff --git a/test/benchmarks/benchmark-08.jsonnet b/test/benchmarks/benchmark-08.jsonnet index 67d38be2..75d7942a 100644 --- a/test/benchmarks/benchmark-08.jsonnet +++ b/test/benchmarks/benchmark-08.jsonnet @@ -4,13 +4,13 @@ local max_number = 100000; driver: { cpp: 'generate_layers', layers: { - event: { total: max_number } - } + event: { total: max_number }, + }, }, sources: { provider: { - cpp: 'benchmarks_provider' - } + cpp: 'benchmarks_provider', + }, }, modules: { a_creator: { @@ -19,17 +19,17 @@ local max_number = 100000; }, even_filter: { cpp: 'accept_even_numbers', - consumes: { creator: 'a_creator', suffix: 'a', layer: 'event' } + consumes: { creator: 'a_creator', suffix: 'a', layer: 'event' }, }, fibonacci_filter: { cpp: 'accept_fibonacci_numbers', - consumes: { creator: 'a_creator', suffix: 'a', layer: "event" }, + consumes: { creator: 'a_creator', suffix: 'a', layer: 'event' }, max_number: max_number, }, d: { cpp: 'verify_even_fibonacci_numbers', experimental_when: ['even_filter:accept_even_numbers', 'fibonacci_filter:accept'], - consumes: { creator: 'a_creator', suffix: 'a', layer: "event" }, + consumes: { creator: 'a_creator', suffix: 'a', layer: 'event' }, max_number: max_number, }, }, diff --git a/test/benchmarks/benchmark-09.jsonnet b/test/benchmarks/benchmark-09.jsonnet index d026935c..4997c557 100644 --- a/test/benchmarks/benchmark-09.jsonnet +++ b/test/benchmarks/benchmark-09.jsonnet @@ -2,31 +2,31 @@ driver: { cpp: 'generate_layers', layers: { - event: { total: 100000 } - } + event: { total: 100000 }, + }, }, sources: { provider: { - cpp: 'benchmarks_provider' - } + cpp: 'benchmarks_provider', + }, }, modules: { a_creator: { cpp: 'last_index', - input: {creator: "a_creator", layer: "event", suffix: "a"} + input: { creator: 'a_creator', layer: 'event', suffix: 'a' }, }, b_creator: { cpp: 'plus_one', - input: {creator: "a_creator", layer: "event", suffix: "a"} + input: { creator: 'a_creator', layer: 'event', suffix: 'a' }, }, even_filter: { cpp: 'accept_even_numbers', - consumes: { creator: 'a_creator', suffix: 'a', layer: "event" } + consumes: { creator: 'a_creator', suffix: 'a', layer: 'event' }, }, d: { cpp: 'read_index', experimental_when: ['even_filter:accept_even_numbers'], - consumes: { creator: 'b_creator', suffix: 'b', layer: "event" } + consumes: { creator: 'b_creator', suffix: 'b', layer: 'event' }, }, }, } diff --git a/test/mock-workflow/G4Stage1.libsonnet b/test/mock-workflow/G4Stage1.libsonnet index c5f1eae7..4df6a8d6 100644 --- a/test/mock-workflow/G4Stage1.libsonnet +++ b/test/mock-workflow/G4Stage1.libsonnet @@ -1,11 +1,11 @@ -local ev = import "event_product.libsonnet"; local generators = import 'SinglesGen.libsonnet'; +local ev = import 'event_product.libsonnet'; { largeant: { cpp: 'largeant', - duration_usec: 156, # Typical: 15662051 - inputs: [ev.creator_event_product(f, "MCTruths") for f in std.objectFields(generators)], - outputs: ["ParticleAncestryMap", "Assns", "SimEnergyDeposits", "AuxDetHits", "MCParticles"], - } + duration_usec: 156, // Typical: 15662051 + inputs: [ev.creator_event_product(f, 'MCTruths') for f in std.objectFields(generators)], + outputs: ['ParticleAncestryMap', 'Assns', 'SimEnergyDeposits', 'AuxDetHits', 'MCParticles'], + }, } diff --git a/test/mock-workflow/G4Stage2.libsonnet b/test/mock-workflow/G4Stage2.libsonnet index dae76be9..73f73fb6 100644 --- a/test/mock-workflow/G4Stage2.libsonnet +++ b/test/mock-workflow/G4Stage2.libsonnet @@ -1,17 +1,17 @@ -local ev = import "event_product.libsonnet"; local g4stage1 = import 'G4Stage1.libsonnet'; +local ev = import 'event_product.libsonnet'; { IonAndScint: { cpp: 'ion_and_scint', - duration_usec: 546, # Typical: 5457973 - inputs: [ev.creator_event_product(f, "SimEnergyDeposits") for f in std.objectFields(g4stage1)], - outputs: ["SimEnergyDeposits", "SimEnergyDeposits_priorSCE"], + duration_usec: 546, // Typical: 5457973 + inputs: [ev.creator_event_product(f, 'SimEnergyDeposits') for f in std.objectFields(g4stage1)], + outputs: ['SimEnergyDeposits', 'SimEnergyDeposits_priorSCE'], }, PDFastSim: { cpp: 'pd_fast_sim', - duration_usec: 69, # Typical: 69681950 + duration_usec: 69, // Typical: 69681950 inputs: [ev.creator_event_product('IonAndScint', 'SimEnergyDeposits_priorSCE')], outputs: ['SimPhotonLites', 'OpDetBacktrackerRecords'], - } + }, } diff --git a/test/mock-workflow/event_product.libsonnet b/test/mock-workflow/event_product.libsonnet index 6f3bb2ac..9cda3758 100644 --- a/test/mock-workflow/event_product.libsonnet +++ b/test/mock-workflow/event_product.libsonnet @@ -1,15 +1,15 @@ { event_product(product):: { - creator: 'input', + creator: 'input', suffix: product, - layer: "event" + layer: 'event', }, creator_event_product(creator, product):: { - creator: creator, + creator: creator, suffix: product, - layer: "event" - } + layer: 'event', + }, } From a23f233231c27f82f2c9b45dc3b74ebcf52b9ca4 Mon Sep 17 00:00:00 2001 From: Beojan Stanislaus Date: Tue, 27 Jan 2026 14:06:34 -0800 Subject: [PATCH 5/7] Use new product_query for edge making --- phlex/core/edge_creation_policy.cpp | 95 ++++++++++++++++------------- phlex/core/edge_creation_policy.hpp | 46 +++++++++----- phlex/core/product_query.cpp | 26 +++++++- phlex/core/product_query.hpp | 11 ++++ phlex/model/type_id.hpp | 18 +++++- 5 files changed, 134 insertions(+), 62 deletions(-) diff --git a/phlex/core/edge_creation_policy.cpp b/phlex/core/edge_creation_policy.cpp index 5a3de719..fad996dd 100644 --- a/phlex/core/edge_creation_policy.cpp +++ b/phlex/core/edge_creation_policy.cpp @@ -3,64 +3,71 @@ #include "fmt/format.h" #include "fmt/ranges.h" #include "spdlog/spdlog.h" +#include #include +#include namespace phlex::experimental { edge_creation_policy::named_output_port const* edge_creation_policy::find_producer( product_query const& query) const { - // TODO: Update later with correct querying - auto [b, e] = producers_.equal_range(query.suffix().value_or("")); - if (b == e) { - spdlog::debug( - "Failed to find an algorithm that creates {} products. Assuming it comes from a provider", - query.suffix().value_or("\"\"")); + auto [creator_b, creator_e] = creator_db_.equal_range(query.creator_hash()); + if (creator_b == creator_e) { + spdlog::debug("Found no matching creators for {}. Assuming it comes from a provider.", + query.to_string()); return nullptr; } - std::map candidates; - for (auto const& [key, producer] : std::ranges::subrange{b, e}) { - // TODO: Definitely not right yet - if (producer.node.plugin() == query.creator() || - producer.node.algorithm() == query.creator()) { - if (query.type() != producer.type) { - spdlog::debug("Matched ({}) from {} but types don't match (`{}` vs `{}`). Excluding " - "from candidate list.", - query.to_string(), - producer.node.full(), - query.type(), - producer.type); - } else { - if (query.type().exact_compare(producer.type)) { - spdlog::debug("Matched ({}) from {} and types match. Keeping in candidate list.", - query.to_string(), - producer.node.full()); - } else { - spdlog::warn("Matched ({}) from {} and types match, but not exactly (produce {} and " - "consume {}). Keeping in candidate list!", - query.to_string(), - producer.node.full(), - query.type().exact_name(), - producer.type.exact_name()); - } - candidates.emplace(producer.node.full(), &producer); - } - } else { - spdlog::error( - "Creator name mismatch between ({}) and {}", query.to_string(), producer.node.full()); - } + std::set matching_creator = std::ranges::subrange(creator_b, creator_e) | std::views::values | + std::ranges::to(); + + auto [type_b, type_e] = type_db_.equal_range(query.type_hash()); + if (type_b == type_e) { + throw std::runtime_error(fmt::format( + "Found no matching types for {} (require {})", query.to_string(), query.type())); } + std::set matching_type = + std::ranges::subrange(type_b, type_e) | std::views::values | std::ranges::to(); - if (candidates.empty()) { - throw std::runtime_error("Cannot identify product matching the query " + query.to_string()); + std::set creator_and_type{}; + std::ranges::set_intersection( + matching_creator, matching_type, std::inserter(creator_and_type, creator_and_type.begin())); + if (creator_and_type.empty()) { + throw std::runtime_error( + fmt::format("Found no creator + type match for {} (required type {})", + query.to_string(), + query.type())); } - if (candidates.size() > 1ull) { - std::string msg = fmt::format("More than one candidate matches the query {}: \n - {}\n", - query.to_string(), - fmt::join(std::views::keys(candidates), "\n - ")); + std::set all_matched; + if (query.suffix()) { + for (auto const* port : creator_and_type) { + if (port->suffix == query.suffix()) { + all_matched.insert(port); + } + } + if (all_matched.empty()) { + throw std::runtime_error( + fmt::format("Of {} creator + type matches for {}, found 0 suffix matches", + creator_and_type.size(), + query.to_string())); + } + } else { + all_matched = std::move(creator_and_type); + } + if (all_matched.size() > 1) { + std::string msg = + fmt::format("Found {} duplicate matches for {}", all_matched.size(), query.to_string()); throw std::runtime_error(msg); } - return candidates.begin()->second; + return *all_matched.begin(); + } + + std::uint64_t edge_creation_policy::hash_string(std::string const& str) + { + using namespace boost::hash2; + xxhash_64 h; + hash_append(h, {}, str); + return h.result(); } } diff --git a/phlex/core/edge_creation_policy.hpp b/phlex/core/edge_creation_policy.hpp index b149e3b6..9764c63e 100644 --- a/phlex/core/edge_creation_policy.hpp +++ b/phlex/core/edge_creation_policy.hpp @@ -2,6 +2,7 @@ #define PHLEX_CORE_EDGE_CREATION_POLICY_HPP #include "phlex/core/message.hpp" +#include "phlex/core/concepts.hpp" #include "phlex/model/product_specification.hpp" #include "phlex/model/type_id.hpp" @@ -9,6 +10,7 @@ #include #include +#include #include namespace phlex::experimental { @@ -20,45 +22,57 @@ namespace phlex::experimental { edge_creation_policy(Args&... producers); struct named_output_port { - algorithm_name node; + std::string suffix; + algorithm_name creator; tbb::flow::sender* port; tbb::flow::sender* to_output; type_id type; }; named_output_port const* find_producer(product_query const& query) const; - auto values() const { return producers_ | std::views::values; } + auto values() const { return producers_; } private: - template - static std::multimap producing_nodes(T& nodes); + // Store a stack of all named_output_ports + std::forward_list producers_; + + // And maps indexing by (hash of) each field + std::multimap creator_db_; + std::multimap suffix_db_; + std::multimap type_db_; - std::multimap producers_; + // Utility to add producers + template + void add_nodes(T& nodes); + static std::uint64_t hash_string(std::string const& str); }; // ============================================================================= // Implementation + template - std::multimap - edge_creation_policy::producing_nodes(T& nodes) + void edge_creation_policy::add_nodes(T& nodes) { - std::multimap result; for (auto const& [node_name, node] : nodes) { - for (auto const& product_name : node->output()) { - if (empty(product_name.name())) - continue; - result.emplace( - product_name.name(), - named_output_port{node_name, &node->sender(), &node->to_output(), product_name.type()}); + for (auto const& spec : node->output()) { + spdlog::debug("Adding product {} with type {}", spec.full(), spec.type()); + auto& port = producers_.emplace_front( + spec.name(), spec.qualifier(), &node->sender(), &node->to_output(), spec.type()); + + // creator_db_ contains entries for plugin name and algorithm name + creator_db_.emplace(hash_string(spec.plugin()), &port); + creator_db_.emplace(hash_string(spec.algorithm()), &port); + + suffix_db_.emplace(hash_string(spec.name()), &port); + type_db_.emplace(spec.type().hash(), &port); } } - return result; } template edge_creation_policy::edge_creation_policy(Args&... producers) { - (producers_.merge(producing_nodes(producers)), ...); + (add_nodes(producers), ...); } } diff --git a/phlex/core/product_query.cpp b/phlex/core/product_query.cpp index edafd118..8dbb4811 100644 --- a/phlex/core/product_query.cpp +++ b/phlex/core/product_query.cpp @@ -1,9 +1,15 @@ #include "phlex/core/product_query.hpp" #include "fmt/format.h" +#include +#include namespace phlex { - void product_query::set_type(experimental::type_id&& type) { type_id_ = std::move(type); } + void product_query::set_type(experimental::type_id&& type) + { + type_id_ = std::move(type); + update_hashes(); + } // Check that all products selected by /other/ would satisfy this query bool product_query::match(product_query const& other) const @@ -59,4 +65,22 @@ namespace phlex { } return experimental::product_specification::create(*suffix()); } + + void product_query::update_hashes() + { + using namespace boost::hash2; + xxhash_64 creator; + hash_append(creator, {}, creator_); + creator_hash_ = creator.result(); + + if (suffix_) { + xxhash_64 suffix; + hash_append(suffix, {}, *suffix_); + suffix_hash_ = suffix.result(); + } + + xxhash_64 type_id; + hash_append(type_id, {}, type_id_); + type_hash_ = type_id.result(); + } } diff --git a/phlex/core/product_query.hpp b/phlex/core/product_query.hpp index a049aaae..a348ecd6 100644 --- a/phlex/core/product_query.hpp +++ b/phlex/core/product_query.hpp @@ -71,6 +71,7 @@ namespace phlex { if (layer_.empty()) { throw std::runtime_error("Cannot specify the empty string as a data layer."); } + update_hashes(); } void set_type(experimental::type_id&& type); @@ -80,6 +81,10 @@ namespace phlex { std::optional const& stage() const noexcept { return stage_; } experimental::type_id const& type() const noexcept { return type_id_; } + std::uint64_t creator_hash() const { return creator_hash_; } + std::uint64_t suffix_hash() const { return suffix_hash_; } + std::uint64_t type_hash() const { return type_hash_; } + // Check that all products selected by /other/ would satisfy this query bool match(product_query const& other) const; @@ -101,6 +106,12 @@ namespace phlex { std::optional suffix_; std::optional stage_; experimental::type_id type_id_; + + std::uint64_t creator_hash_ = 0; + std::uint64_t suffix_hash_ = 0; + std::uint64_t type_hash_ = 0; + + void update_hashes(); }; using product_queries = std::vector; diff --git a/phlex/model/type_id.hpp b/phlex/model/type_id.hpp index 080f3e88..c043fcce 100644 --- a/phlex/model/type_id.hpp +++ b/phlex/model/type_id.hpp @@ -7,7 +7,8 @@ #include "fmt/format.h" #include "fmt/ranges.h" #include -#include +#include +#include #include #include @@ -44,6 +45,18 @@ namespace phlex::experimental { constexpr builtin fundamental() const { return static_cast(id_ & 0x0F); } + std::uint64_t hash() { + if (hash_ != 0) { + // Yes, there's a *very* small chance of a conflict + return hash_; + } + + boost::hash2::xxhash_64 h; + boost::hash2::hash_append(h, {}, *this); + hash_ = h.result(); + return hash_; + } + template friend constexpr void tag_invoke(boost::hash2::hash_append_tag const&, Provider const&, @@ -89,6 +102,9 @@ namespace phlex::experimental { // This is used only if the product type is a struct std::vector children_; + + // Hash + std::uint64_t hash_ = 0; }; using type_ids = std::vector; From bbc34e0976a1c599601b3314b94c125300252299 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 22:12:10 +0000 Subject: [PATCH 6/7] Apply clang-format fixes --- phlex/core/edge_creation_policy.hpp | 2 +- phlex/model/type_id.hpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/phlex/core/edge_creation_policy.hpp b/phlex/core/edge_creation_policy.hpp index 9764c63e..4f354aed 100644 --- a/phlex/core/edge_creation_policy.hpp +++ b/phlex/core/edge_creation_policy.hpp @@ -1,8 +1,8 @@ #ifndef PHLEX_CORE_EDGE_CREATION_POLICY_HPP #define PHLEX_CORE_EDGE_CREATION_POLICY_HPP -#include "phlex/core/message.hpp" #include "phlex/core/concepts.hpp" +#include "phlex/core/message.hpp" #include "phlex/model/product_specification.hpp" #include "phlex/model/type_id.hpp" diff --git a/phlex/model/type_id.hpp b/phlex/model/type_id.hpp index c043fcce..a108caa4 100644 --- a/phlex/model/type_id.hpp +++ b/phlex/model/type_id.hpp @@ -45,7 +45,8 @@ namespace phlex::experimental { constexpr builtin fundamental() const { return static_cast(id_ & 0x0F); } - std::uint64_t hash() { + std::uint64_t hash() + { if (hash_ != 0) { // Yes, there's a *very* small chance of a conflict return hash_; From eafdb22f3167e920d1a47c0f071f49fcda624ab3 Mon Sep 17 00:00:00 2001 From: Beojan Stanislaus Date: Wed, 28 Jan 2026 11:13:52 -0800 Subject: [PATCH 7/7] Use utilities::hash in product_query, after updating it to use Boost hash2 --- phlex/core/product_query.cpp | 18 ++++-------------- phlex/utilities/hashing.cpp | 22 +++++++++++++++++----- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/phlex/core/product_query.cpp b/phlex/core/product_query.cpp index 8dbb4811..ca4ceab7 100644 --- a/phlex/core/product_query.cpp +++ b/phlex/core/product_query.cpp @@ -1,8 +1,7 @@ #include "phlex/core/product_query.hpp" +#include "phlex/utilities/hashing.hpp" #include "fmt/format.h" -#include -#include namespace phlex { void product_query::set_type(experimental::type_id&& type) @@ -68,19 +67,10 @@ namespace phlex { void product_query::update_hashes() { - using namespace boost::hash2; - xxhash_64 creator; - hash_append(creator, {}, creator_); - creator_hash_ = creator.result(); - + creator_hash_ = experimental::hash(creator_); + type_hash_ = type_id_.hash(); if (suffix_) { - xxhash_64 suffix; - hash_append(suffix, {}, *suffix_); - suffix_hash_ = suffix.result(); + suffix_hash_ = experimental::hash(*suffix_); } - - xxhash_64 type_id; - hash_append(type_id, {}, type_id_); - type_hash_ = type_id.result(); } } diff --git a/phlex/utilities/hashing.cpp b/phlex/utilities/hashing.cpp index 4c0964ab..7578c5dc 100644 --- a/phlex/utilities/hashing.cpp +++ b/phlex/utilities/hashing.cpp @@ -1,11 +1,17 @@ #include "phlex/utilities/hashing.hpp" -#include "boost/functional/hash.hpp" +#include +#include +#include namespace phlex::experimental { - std::hash const string_hasher{}; - - std::size_t hash(std::string const& str) { return string_hasher(str); } + std::size_t hash(std::string const& str) + { + using namespace boost::hash2; + xxhash_64 h; + hash_append(h, {}, str); + return h.result(); + } std::size_t hash(std::size_t i) noexcept { return i; } @@ -15,5 +21,11 @@ namespace phlex::experimental { return i; } - std::size_t hash(std::size_t i, std::string const& str) { return hash(i, hash(str)); } + std::size_t hash(std::size_t i, std::string const& str) + { + using namespace boost::hash2; + xxhash_64 h{i}; + hash_append(h, {}, str); + return h.result(); + } }