gp/src/problem.cc

Go to the documentation of this file.

 
#include <set>
 
#include "kernel/gp/src/problem.h"
#include "kernel/gp/src/constant.h"
#include "kernel/gp/src/evaluator.h"
#include "kernel/gp/src/lambda_f.h"
#include "kernel/gp/src/variable.h"
 
#include "third_party/tinyxml2/tinyxml2.h"
 
namespace vita
{
 
const src_problem::default_symbols_t src_problem::default_symbols = {};
 
namespace detail
{
template<class C>
std::set<std::vector<C>> seq_with_rep(const std::set<C> &availables,
                                      std::size_t size)
{
  Expects(availables.size());
  Expects(size);
 
  std::set<std::vector<C>> ret;
 
  std::function<void (std::size_t, std::vector<C>)> swr;
  swr = [&](std::size_t left, std::vector<C> current)
        {
          if (!left)  // we have a sequence of the correct length
          {
            ret.insert(current);
            return;
          }
 
          for (auto elem : availables)
          {
            current.push_back(elem);
            swr(left - 1, current);
            current.pop_back();
          }
        };
 
  swr(size, {});
  return ret;
}
}  // namespace detail
 
src_problem::src_problem() : problem(), training_(), validation_(), factory_()
{
}
 
src_problem::src_problem(const std::filesystem::path &ds, typing t)
  : src_problem()
{
  vitaINFO << "Reading dataset " << ds << "...";
  data(dataset_t::training).read(ds);
 
  vitaINFO << "...dataset read. Examples: " << data(dataset_t::training).size()
           << ", categories: " << categories()
           << ", features: " << variables()
           << ", classes: " << classes();
 
  setup_terminals(t);
}
 
src_problem::src_problem(std::istream &ds, typing t) : src_problem()
{
  vitaINFO << "Reading dataset from input stream...";
  data(dataset_t::training).read_csv(ds);
 
  vitaINFO << "...dataset read. Examples: " << data(dataset_t::training).size()
           << ", categories: " << categories()
           << ", features: " << variables()
           << ", classes: " << classes();
 
  setup_terminals(t);
}
 
src_problem::src_problem(const std::filesystem::path &ds,
                         const default_symbols_t &, typing t)
  : src_problem(ds, std::filesystem::path(), t)
{
}
 
src_problem::src_problem(const std::filesystem::path &ds,
                         const std::filesystem::path &symbols, typing t)
  : src_problem()
{
  vitaINFO << "Reading dataset " << ds << "...";
  data(dataset_t::training).read(ds);
 
  vitaINFO << "....dataset read. Examples: " << data(dataset_t::training).size()
           << ", categories: " << categories()
           << ", features: " << variables()
           << ", classes: " << classes();
 
  setup_symbols(symbols, t);
}
 
bool src_problem::operator!() const
{
  return !training_.size() || !sset.enough_terminals();
}
 
void src_problem::setup_terminals(typing t)
{
  vitaINFO << "Setting up terminals...";
 
  const auto &columns(training_.columns);
  if (columns.size() < 2)
    throw exception::insufficient_data("Cannot generate the terminal set");
 
  std::string variables;
 
  category_set categories(training_.columns, t);
  for (std::size_t i(1); i < columns.size(); ++i)
  {
    // Sets up the variables (features).
    const auto provided_name(columns[i].name);
    const auto name(provided_name.empty() ? "X" + std::to_string(i)
                                          : provided_name);
    const category_t category(categories.column(i).category);
 
    if (insert<variable>(name, static_cast<unsigned>(i - 1), category))
      variables += " `" + name + "`";
 
    // Sets up states for nominal attributes.
    for (const auto &s : columns[i].states)
      switch (columns[i].domain)
      {
      case d_double:
        insert<constant<D_DOUBLE>>(std::get<D_DOUBLE>(s), category);
        break;
      case d_int:
        insert<constant<D_INT>>(std::get<D_INT>(s), category);
        break;
      case d_string:
        insert<constant<D_STRING>>(std::get<D_STRING>(s), category);
        break;
      default:
        exception::insufficient_data("Cannot generate the terminal set");
      }
 
    /*std::visit([this, category](const auto &cs)
                 {
                   using T = std::decay_t<decltype(cs)>>;
                   insert<constant<T>(cs, category);
                   });*/
  }
 
  vitaINFO << "...terminals ready. Variables:" << variables;
}
 
std::size_t src_problem::setup_symbols(typing t)
{
  return setup_symbols({}, t);
}
 
std::size_t src_problem::setup_symbols(const std::filesystem::path &file,
                                       typing t)
{
  sset.clear();
 
  setup_terminals(t);
 
  return file.empty() ? setup_symbols_impl() : setup_symbols_impl(file);
}
 
std::size_t src_problem::setup_symbols_impl()
{
  vitaINFO << "Setting up default symbol set...";
 
  category_set categories(training_.columns);
  const auto used_categories(categories.used_categories());
  std::size_t inserted(0);
 
  for (const auto &category : used_categories)
    if (compatible({category}, {"numeric"}, categories))
    {
      const std::vector<std::string> base(
        {"1.0", "2.0", "3.0", "4.0", "5.0", "6.0", "7.0", "8.0", "9.0",
         "FABS", "FADD", "FDIV", "FLN", "FMUL", "FMOD", "FSUB"});
 
      for (const auto &s: base)
        if (sset.insert(factory_.make(s, {category})))
          ++inserted;
    }
    else if (compatible({category}, {"string"}, categories))
    {
      if (sset.insert(factory_.make("SIFE", {category, 0})))
        ++inserted;
    }
 
  vitaINFO << "...default symbol set ready. Symbols: " << inserted;
  return inserted;
}
 
std::size_t src_problem::setup_symbols_impl(const std::filesystem::path &file)
{
  vitaINFO << "Reading symbol set " << file << "...";
  tinyxml2::XMLDocument doc;
  if (doc.LoadFile(file.string().c_str()) != tinyxml2::XML_SUCCESS)
    throw exception::data_format("Symbol set format error");
 
  category_set categories(training_.columns);
  const auto used_categories(categories.used_categories());
  std::size_t parsed(0);
 
  // When I wrote this, only God and I understood what I was doing.
  // Now, God only knows.
  tinyxml2::XMLHandle handle(&doc);
  auto *symbolset(handle.FirstChildElement("symbolset").ToElement());
 
  if (!symbolset)
    throw exception::data_format("Empty symbol set");
 
  for (auto *s(symbolset->FirstChildElement("symbol"));
       s;
       s = s->NextSiblingElement("symbol"))
  {
    if (!s->Attribute("name"))
    {
      vitaERROR << "Skipped unnamed symbol in symbolset";
      continue;
    }
    const std::string sym_name(s->Attribute("name"));
 
    if (const char *sym_sig = s->Attribute("signature")) // single category,
    {                                                    // uniform init
      for (auto category : used_categories)
        if (compatible({category}, {std::string(sym_sig)}, categories))
        {
          const auto n_args(factory_.args(sym_name));
          std::string signature(sym_name + ":");
 
          for (std::size_t j(0); j < n_args; ++j)
            signature += " " + std::to_string(category);
          vitaDEBUG << "Adding to symbol set " << signature;
 
          sset.insert(factory_.make(sym_name, cvect(n_args, category)));
        }
    }
    else  // !sym_sig => complex signature
    {
      auto *sig(s->FirstChildElement("signature"));
      if (!sig)
      {
        vitaERROR << "Skipping " << sym_name << " symbol (empty signature)";
        continue;
      }
 
      std::vector<std::string> args;
      for (auto *arg(sig->FirstChildElement("arg"));
           arg;
           arg = arg->NextSiblingElement("arg"))
      {
        if (!arg->GetText())
        {
          vitaERROR << "Skipping " << sym_name << " symbol (wrong signature)";
          args.clear();
          break;
        }
 
        args.push_back(arg->GetText());
      }
 
      // From the list of all the sequences with repetition of `args.size()`
      // elements...
      const auto sequences(detail::seq_with_rep(used_categories, args.size()));
 
      // ...we choose those compatible with the xml signature of the current
      // symbol.
      for (const auto &seq : sequences)
        if (compatible(seq, args, categories))
        {
          std::string signature(sym_name + ":");
          for (const auto &j : seq)
            signature += " " + std::to_string(j);
          vitaDEBUG << "Adding to symbol set " << signature;
 
          sset.insert(factory_.make(sym_name, seq));
        }
    }
 
    ++parsed;
  }
 
  vitaINFO << "...symbol set read. Symbols: " << parsed;
  return parsed;
}
 
bool src_problem::compatible(const cvect &instance,
                             const std::vector<std::string> &pattern,
                             const category_set &categories) const
{
  Expects(instance.size() == pattern.size());
 
  const auto sup(instance.size());
  for (std::size_t i(0); i < sup; ++i)
  {
    const std::string p_i(pattern[i]);
    const bool generic(from_weka(p_i) != d_void);
 
    if (generic)  // numeric, string, integer...
    {
      if (categories.category(instance[i]).domain != from_weka(p_i))
        return false;
    }
    else
    {
      if (instance[i] != categories.column(p_i).category)
        return false;
    }
  }
 
  return true;
}
 
unsigned src_problem::categories() const
{
  return static_cast<unsigned>(sset.categories());
}
 
unsigned src_problem::classes() const
{
  return static_cast<unsigned>(training_.classes());
}
 
unsigned src_problem::variables() const
{
  return training_.variables();
}
 
dataframe &src_problem::data(dataset_t t)
{
  return t == dataset_t::training ? training_ : validation_;
}
 
const dataframe &src_problem::data(dataset_t t) const
{
  return t == dataset_t::training ? training_ : validation_;
}
 
bool src_problem::is_valid() const
{
  return problem::is_valid();
}
 
}  // namespace vita