gp/src/evaluator.tcc

Go to the documentation of this file.

 
#if !defined(ULTRA_SRC_EVALUATOR_H)
#  error "Don't include this file directly, include the specific .h instead"
#endif
 
#if !defined(ULTRA_SRC_EVALUATOR_TCC)
#define      ULTRA_SRC_EVALUATOR_TCC

template<EvaluationDataset D>
evaluator<D>::evaluator(D &d) noexcept : dat_(&d)
{
}

template<EvaluationDataset D>
const auto &evaluator<D>::data() const noexcept
{
  if constexpr (is_multi_dataset_v<D>)
    return dat_->selected();
  else
    return *dat_;
}

template<EvaluationDataset D>
bool evaluator<D>::set_parallel_evaluation(unsigned helpers,
                                           std::size_t min_samples) noexcept
{
  helpers_ = helpers;
  min_samples_ = !min_samples ? default_min_samples_per_thread : min_samples;
  return true;
}

template<EvaluationDataset D>
unsigned evaluator<D>::parallel_helpers() const noexcept
{
  return helpers_;
}

template<EvaluationDataset D>
std::size_t evaluator<D>::min_samples_per_thread() const noexcept
{
  return min_samples_;
}

template<Individual P, class F, class D, aggregation_mode A, evaluation_mode M>
requires ExampleEvaluator<F, D, P>
aggregate_evaluator<P, F, D, A, M>::aggregate_evaluator(D &d) noexcept
  : evaluator<D>(d)
{
}

template<Individual P, class F, class D, aggregation_mode A, evaluation_mode M>
requires ExampleEvaluator<F, D, P>
bool aggregate_evaluator<P, F, D, A, M>::set_parallel_evaluation(
  unsigned n, std::size_t min_samples) noexcept
  requires parallel_safe_example_evaluator<F>
{
  return evaluator<D>::set_parallel_evaluation(n, min_samples);
}

template<Individual P, class F, class D, aggregation_mode A, evaluation_mode M>
requires ExampleEvaluator<F, D, P>
double aggregate_evaluator<P, F, D, A, M>::operator()(const P &prg) const
{
  return eval_impl(prg, 1);
}

template<Individual P, class F, class D, aggregation_mode A, evaluation_mode M>
requires ExampleEvaluator<F, D, P>
double aggregate_evaluator<P, F, D, A, M>::fast(const P &prg) const
{
  const auto n(std::ranges::distance(this->data()));
  return eval_impl(prg, n >= fast_min_examples ? fast_step : 1);
}

template<Individual P, class F, class D, aggregation_mode A, evaluation_mode M>
requires ExampleEvaluator<F, D, P>
auto aggregate_evaluator<P, F, D, A, M>::predictor(const P &prg) const
  requires internal::has_regression_predictor_v<F, P>
{
  return reg_oracle<P>(prg);
}

template<Individual P, class F, class D, aggregation_mode A, evaluation_mode M>
requires ExampleEvaluator<F, D, P>
double aggregate_evaluator<P, F, D, A, M>::eval_impl(const P &prg,
                                                     std::ptrdiff_t step) const
{
  const auto &dat(this->data());
  const auto examples(std::ranges::distance(dat));
  Expects(0 < step && step <= examples);
 
  const auto end(std::end(dat));
  const auto sampled_count(static_cast<std::size_t>((examples - 1) / step + 1));
 
  std::size_t workers(1);
  if constexpr (std::ranges::forward_range<decltype(dat)>
                && parallel_safe_example_evaluator<F>)
    if (this->parallel_helpers())
      workers = std::clamp(sampled_count / this->min_samples_per_thread(),
                           1uz, this->parallel_helpers() + 1uz);
 
  const auto stride(step * static_cast<std::ptrdiff_t>(workers));
 
  struct partial
  {
    double mean {0.0};
    std::size_t sampled {0};
    std::size_t valid {0};
 
    void merge(const partial &part)
    {
      sampled += part.sampled;
      if (!part.valid)
        return;
 
      if (valid)
        mean += (part.mean - mean)
                * (static_cast<double>(part.valid)
                   / static_cast<double>(valid + part.valid));
      else
        mean = part.mean;
 
      valid += part.valid;
    }
  };
 
  const auto evaluate_strip([&](std::size_t strip)
  {
    const F f(prg);  // each strip owns its oracle
    partial result;
 
    for (auto it(std::ranges::next(std::begin(dat),
                                   static_cast<std::ptrdiff_t>(strip) * step, end));
         it != end;
         std::ranges::advance(it, stride, end))
    {
      ++result.sampled;
 
      if (const std::optional<double> value(f(*it)); value)
        result.mean += (*value - result.mean)
                       / static_cast<double>(++result.valid);
    }
 
    return result;
  });
 
  std::vector<std::future<partial>> futures;
  futures.reserve(workers - 1);
 
  for (std::size_t strip(1); strip < workers; ++strip)
    futures.push_back(std::async(std::launch::async, evaluate_strip, strip));
 
  partial total(evaluate_strip(0));  // caller handles the first strip
 
  for (auto &future : futures)
    total.merge(future.get());
 
  Ensures(0 < total.sampled
          && total.sampled <= static_cast<std::size_t>(examples));
 
  if (!total.valid)
    return std::numeric_limits<double>::lowest();
 
  double acc(total.mean);
 
  if (total.valid < total.sampled)
  {
    const auto missing_per_valid(static_cast<double>(total.sampled - total.valid)
                                 / static_cast<double>(total.valid));
 
    if constexpr (M == evaluation_mode::error)
      acc += missing_per_valid * (acc + 1.0);
    else
      acc -= missing_per_valid * (std::fabs(acc) + 1.0);
  }
 
  if constexpr (A == aggregation_mode::sum)
    acc *= static_cast<double>(examples);
 
  if constexpr (M == evaluation_mode::error)
    acc = -acc;
 
  return acc;
}

template<Individual P>
mae_error_functor<P>::mae_error_functor(const P &prg) : oracle_(prg)
{
}

template<Individual P>
std::optional<double>
mae_error_functor<P>::operator()(const example &example) const
{
  if (const auto foreseen(oracle_(example.input)); has_value(foreseen))
    return std::fabs(lexical_cast<D_DOUBLE>(foreseen)
                     - label_as<D_DOUBLE>(example));
 
  return std::nullopt;
}

template<Individual P>
rmae_error_functor<P>::rmae_error_functor(const P &prg) : oracle_(prg)
{
}

template<Individual P>
std::optional<double>
rmae_error_functor<P>::operator()(const example &example) const
{
  if (const auto foreseen(oracle_(example.input)); has_value(foreseen))
  {
    const auto approx(lexical_cast<D_DOUBLE>(foreseen));
    const auto target(label_as<D_DOUBLE>(example));
 
    const auto delta(std::fabs(target - approx));
 
    // Avoid `0 / 0` when both values are zero (uses an inequality, which never
    // triggers a warning on any compiler. Mathematically identical to
    // `delta==0.0` because `delta` is non-negative).
    if (delta <= 0.0)
      return 0.0;
 
    return delta / (std::fabs(approx) + std::fabs(target));
    // Some alternatives for the error:
    // * delta / std::max(approx, target)
    // * delta / std::fabs(target)
    //
    // The chosen formula seems numerically more stable and gives a result
    // in a limited range of values.
  }
 
  return std::nullopt;
}

template<Individual P>
mse_error_functor<P>::mse_error_functor(const P &prg) : oracle_(prg)
{
}

template<Individual P>
std::optional<double>
mse_error_functor<P>::operator()(const example &example) const
{
  if (const auto foreseen(oracle_(example.input)); has_value(foreseen))
  {
    const double err(lexical_cast<D_DOUBLE>(foreseen)
                     - label_as<D_DOUBLE>(example));
    return err * err;
  }
 
  return std::nullopt;
}

template<Individual P>
count_error_functor<P>::count_error_functor(const P &prg) : oracle_(prg)
{
}

template<Individual P>
double count_error_functor<P>::operator()(const example &example) const
{
  const auto foreseen(oracle_(example.input));
 
  if (!has_value(foreseen))
    return 1.0;
 
  if (!issmall(lexical_cast<D_DOUBLE>(foreseen) - label_as<D_DOUBLE>(example)))
    return 1.0;
 
  return 0.0;
}

template<Individual P>
gaussian_evaluator<P>::gaussian_evaluator(multi_dataset<dataframe> &d)
  : evaluator(d)
{
}

template<Individual P>
double gaussian_evaluator<P>::operator()(const P &prg) const
{
  const auto &dat(this->data());
  Expects(dat.classes() >= 2);
  basic_gaussian_oracle<P, false> oracle(prg, dat);
 
  double d(0.0);
  for (auto &example : dat)
    if (const auto res(oracle.tag(example.input)); res.label == label(example))
    {
      const auto scale(static_cast<double>(dat.classes() - 1));
      // Note:
      // * `(1.0 - res.sureness)` is the sum of the errors;
      // * `(res.sureness - 1.0)` is the opposite (standardized fitness);
      // * `(res.sureness - 1.0) / scale` is the opposite of the average error.
      d += (res.sureness - 1.0) / scale;
    }
    else
    {
      // Note:
      // * the maximum single class error is `1.0`;
      // * the maximum average class error is `1.0 / dat.classes()`;
      // So `-1.0` is like to say that we have a complete failure.
      d -= 1.0;
    }
 
  return d;
}

template<Individual P>
auto gaussian_evaluator<P>::predictor(const P &prg) const
{
  return gaussian_oracle<P>(prg, this->data());
}

template<Individual P>
binary_evaluator<P>::binary_evaluator(multi_dataset<dataframe> &d)
  : evaluator(d)
{
}

template<Individual P>
double binary_evaluator<P>::operator()(const P &prg) const
{
  const auto &dat(this->data());
  Expects(dat.classes() == 2);
  basic_binary_oracle<P, false> oracle(prg, dat);
 
  double err(0.0);
 
  for (auto &example : dat)
    if (const auto res(oracle.tag(example.input)); res.label != label(example))
      err += 1.0 + res.sureness;
 
  return -err;
}

template<Individual P>
auto binary_evaluator<P>::predictor(const P &prg) const
{
  return binary_oracle<P>(prg, this->data());
}
 
#endif  // include guard