Dyna.hh

Go to the documentation of this file.

#ifndef _DYNA_HH_
#define _DYNA_HH_

#include <rl_common/Random.h>
#include <rl_common/core.hh>

#include <map>
#include <set>
#include <vector>

class Dyna: public Agent {
public:
  Dyna(int numactions, float gamma,
       float initialvalue, float alpha, int k, float epsilon,
           Random rng = Random());

  Dyna(const Dyna &);

  virtual ~Dyna();

  virtual int first_action(const std::vector<float> &s);
  virtual int next_action(float r, const std::vector<float> &s);
  virtual void last_action(float r);
  virtual void setDebug(bool d);
  virtual void seedExp(std::vector<experience>);
  virtual void savePolicy(const char* filename);

  
  int getBestAction(const std::vector<float> &s);
  double getSeconds();

  void printState(const std::vector<float> &s);
  float getValue(std::vector<float> state);
  
  std::vector<float>::iterator random_max_element(
                                                   std::vector<float>::iterator start,
                                                   std::vector<float>::iterator end);

  void logValues(ofstream *of, int xmin, int xmax, int ymin, int ymax);
  float epsilon;

protected:
  typedef const std::vector<float> *state_t;

  struct dynaExperience {
    state_t s;
    state_t next;
    float r;
    bool term;
    int a;
  };
  void addExperience(float r, state_t s, bool term);

  state_t canonicalize(const std::vector<float> &s);

private:
  std::set<std::vector<float> > statespace;

  std::map<state_t, std::vector<float> > Q;

  std::vector<dynaExperience> experiences;

  const int numactions;
  const float gamma;

  const float initialvalue;
  const float alpha;
  const int k;

  Random rng;
  float *currentq;
  state_t laststate;
  int lastact;

  bool ACTDEBUG;
};

#endif