Ultra 0.9.4
Loading...
Searching...
No Matches
gp/src/evaluator.h
Go to the documentation of this file.
1
12
13#if !defined(ULTRA_SRC_EVALUATOR_H)
14#define ULTRA_SRC_EVALUATOR_H
15
16#include "kernel/evaluator.h"
20
21#include <future>
22#include <optional>
23#include <vector>
24
25namespace ultra::src
26{
30
40/// regardless of whether a single dataset or a multi-dataset container is
41/// used.
42///
43template<class D> concept EvaluationDataset =
45 || DataSet<D>;
46
47
59/// of the returned value. The aggregation semantics are defined by the
60/// evaluator using the functor (e.g. sum vs average, error vs score).
61///
62/// Accepts both functors returning `std::optional<double>` and functors
63/// returning `double`.
64///
65/// \note
66/// The functor is typically constructed from an individual/program and may
67/// internally use an oracle or interpreter to compute predictions.
68///
69template<class F, class D, class P> concept ExampleEvaluator =
71 && std::invocable<F, internal::dataset_example_t<D>>
72 && std::convertible_to<std::invoke_result_t<F, internal::dataset_example_t<D>>,
73 std::optional<double>>;
74
77
81template<class F> inline constexpr bool parallel_safe_example_evaluator = false;
82
85///
86/// \tparam D dataset type
87///
88/// This class stores a reference to a dataset and provides a uniform access
89/// interface for both plain datasets and `multi_dataset` specialisations.
90/// It is intended to be used as a protected base class for concrete
91/// evaluators.
92///
93template<EvaluationDataset D>
95{
96public:
97 static constexpr std::size_t default_min_samples_per_thread = 512;
98
99protected:
100 explicit evaluator(D &) noexcept;
101
102 bool set_parallel_evaluation(unsigned, std::size_t = 0) noexcept;
103
104 [[nodiscard]] const auto &data() const noexcept;
105 [[nodiscard]] std::size_t min_samples_per_thread() const noexcept;
106 [[nodiscard]] unsigned parallel_helpers() const noexcept;
107
108private:
109 D *dat_ {nullptr};
110 unsigned helpers_ {0};
111 std::size_t min_samples_ {default_min_samples_per_thread};
112};
113
114enum class aggregation_mode { sum, average };
115enum class evaluation_mode { error, score };
116
117template<Individual P, class F, class D, aggregation_mode A, evaluation_mode M>
118requires ExampleEvaluator<F, D, P>
120{
121public:
122 static constexpr std::ptrdiff_t fast_step = 5;
123
124 explicit aggregate_evaluator(D &) noexcept;
125
126 bool set_parallel_evaluation(unsigned, std::size_t = 0) noexcept
128
129 [[nodiscard]] double operator()(const P &) const;
130 [[nodiscard]] double fast(const P &) const;
131
132 [[nodiscard]] auto predictor(const P &) const
133 requires internal::has_regression_predictor_v<F, P>;
134
135private:
136 static constexpr std::ptrdiff_t fast_min_examples = 100;
137
138 [[nodiscard]] double eval_impl(const P &, std::ptrdiff_t) const;
139};
140
152/// - higher is better;
153/// - values are comparable between full and fast evaluation modes.
154///
155/// \see
156/// mse_evaluator, mae_evaluator, rmae_evaluator, count_error_evaluator
157///
158template<Individual P, class F, class D = multi_dataset<dataframe>>
160 aggregation_mode::average,
161 evaluation_mode::error>;
162
163template<Individual P, class F, class D = multi_dataset<dataframe>>
164using sum_error_evaluator = aggregate_evaluator<P, F, D,
165 aggregation_mode::sum,
166 evaluation_mode::error>;
167
168template<Individual P, class F, class D = multi_dataset<dataframe>>
169using avg_score_evaluator = aggregate_evaluator<P, F, D,
170 aggregation_mode::average,
171 evaluation_mode::score>;
173template<Individual P, class F, class D = multi_dataset<dataframe>>
174using sum_score_evaluator = aggregate_evaluator<P, F, D,
175 aggregation_mode::sum,
176 evaluation_mode::score>;
177
190template<Individual P>
191class mae_error_functor
192{
193public:
194 mae_error_functor(const P &);
195
196 [[nodiscard]] std::optional<double> operator()(const example &) const;
197
198private:
200};
201
207template<Individual P>
209
236template<Individual P>
238{
239public:
240 explicit rmae_error_functor(const P &);
241
242 [[nodiscard]] std::optional<double> operator()(const example &) const;
243
244private:
246};
247
253template<Individual P>
255
281template<Individual P>
283{
284public:
285 explicit mse_error_functor(const P &);
286
287 [[nodiscard]] std::optional<double> operator()(const example &) const;
288
289private:
291};
292
298template<Individual P>
300
309
316template<Individual P>
318{
319public:
320 explicit count_error_functor(const P &);
321
322 [[nodiscard]] double operator()(const example &) const;
323
324private:
326};
327
328template<Individual P>
329inline constexpr bool parallel_safe_example_evaluator<mae_error_functor<P>> = true;
330template<Individual P>
332template<Individual P>
333inline constexpr bool parallel_safe_example_evaluator<mse_error_functor<P>> = true;
334template<Individual P>
336
343template<Individual P>
344using count_error_evaluator = sum_error_evaluator<P, count_error_functor<P>>;
345
359template<Individual P>
360class gaussian_evaluator : public evaluator<multi_dataset<dataframe>>
361{
362public:
364
365 [[nodiscard]] double operator()(const P &) const;
366 [[nodiscard]] auto predictor(const P &) const;
367};
368
370
377template<Individual P>
378class binary_evaluator : public evaluator<multi_dataset<dataframe>>
379{
380public:
382
383 [[nodiscard]] double operator()(const P &) const;
384 [[nodiscard]] auto predictor(const P &) const;
385};
386
387namespace internal
388{
389template<Individual P>
390struct has_regression_predictor<mae_error_functor<P>, P> : std::true_type {};
391
392template<Individual P>
393struct has_regression_predictor<rmae_error_functor<P>, P> : std::true_type {};
394
395template<Individual P>
396struct has_regression_predictor<mse_error_functor<P>, P> : std::true_type {};
397} // namespace internal
398
400
401} // namespace ultra::src
402
403#endif // include guard
Definition gp/src/evaluator.h:120
aggregate_evaluator(D &) noexcept
Definition gp/src/evaluator.h:97
double operator()(const P &) const
Definition gp/src/evaluator.h:129
Definition oracle.h:111
binary_evaluator(multi_dataset< dataframe > &)
Definition gp/src/evaluator.h:496
double operator()(const P &) const
Definition gp/src/evaluator.h:510
auto predictor(const P &) const
Definition gp/src/evaluator.h:532
count_error_functor(const P &)
Definition gp/src/evaluator.h:407
double operator()(const example &) const
Definition gp/src/evaluator.h:418
Definition dataframe.h:76
const auto & data() const noexcept
Definition gp/src/evaluator.h:40
bool set_parallel_evaluation(unsigned, std::size_t=0) noexcept
Definition gp/src/evaluator.h:59
evaluator(D &) noexcept
Definition gp/src/evaluator.h:27
std::size_t min_samples_per_thread() const noexcept
Definition gp/src/evaluator.h:85
unsigned parallel_helpers() const noexcept
Definition gp/src/evaluator.h:73
double operator()(const P &) const
Definition gp/src/evaluator.h:449
auto predictor(const P &) const
Definition gp/src/evaluator.h:485
gaussian_evaluator(multi_dataset< dataframe > &)
Definition gp/src/evaluator.h:437
Definition gp/src/evaluator.h:192
std::optional< double > operator()(const example &) const
Definition gp/src/evaluator.h:389
mse_error_functor(const P &)
Definition gp/src/evaluator.h:377
Definition multi_dataset.h:50
Definition gp/src/evaluator.h:238
rmae_error_functor(const P &)
Definition gp/src/evaluator.h:332
std::optional< double > operator()(const example &) const
Definition gp/src/evaluator.h:344
Definition individual.h:93
Definition multi_dataset.h:37
Definition gp/src/evaluator.h:43
Definition gp/src/evaluator.h:69
constexpr bool parallel_safe_example_evaluator
Definition gp/src/evaluator.h:81
avg_error_evaluator< P, rmae_error_functor< P > > rmae_evaluator
Definition gp/src/evaluator.h:254
sum_error_evaluator< P, count_error_functor< P > > count_error_evaluator
Definition gp/src/evaluator.h:344
avg_error_evaluator< P, mse_error_functor< P > > mse_evaluator
Definition gp/src/evaluator.h:299
aggregate_evaluator< P, F, D, aggregation_mode::average, evaluation_mode::error > avg_error_evaluator
Definition gp/src/evaluator.h:159
avg_error_evaluator< P, mae_error_functor< P > > mae_evaluator
Definition gp/src/evaluator.h:208
constexpr bool is_multi_dataset_v
Convenience variable template for is_multi_dataset.
Definition multi_dataset.h:89
STL namespace.
Definition dataframe.h:54