/**
 * GQ(lambda): off-policy learning algorithm
 * @author Adam White, converted to C++ by Rupam Mahmood and Rich Sutton
 */

class GQlambda
{
  double *theta; //learning weights
  double *w;     //learning weights
  double *e;     //elegibility trace vector
  int n;         //dimensionality of the vectors

public:

  GQlambda(int nn) {
    n = nn;
    theta = new double[n];
    w = new double[n];
    e = new double[n];
    for (int i=0; i<n; i++) theta[i]=w[i]=e[i]=0;
  }

  /**
   * Inputs::
   *   phi - feature vector corresponding to action A_t in state S_t
   *   phi_bar - expected next state feature vector corresponding to a \in A and S_t+1
   *   lambda - elegibility trace parameter [0,1]
   *   gamma - discount factor [0,1]
   *   R - transient reward
   *   rho - ratio of target policy to behaviour policy [0,1]
   *   I - set of interest for S_t, S_t [0,1]
   **/

  void learn(double phi[], double phi_bar[], double lambda, double gamma, double R, double rho, double I)
  {
    double alpha = 0.0001, eta = 1.0; //step size parameters
    double delta, dot_w_phi, dot_w_e;

    delta = R + gamma*dot(theta,phi_bar) - dot(theta,phi);

    for (int i=0; i<n; i++)
      e[i] = rho*e[i] + I*phi[i];

    dot_w_e = dot(w,e);
    dot_w_phi = dot(w,phi);

    for (int i=0; i<n; i++) {
      theta[i] += alpha * (delta*e[i] - gamma*(1-lambda)*dot_w_e*phi_bar[i]);
      w[i] += alpha*eta * (delta*e[i] - dot_w_phi*phi[i]);
      e[i] *= gamma*lambda;
    }
  }

  double dot(double v1[], double v2[]) {
    double sum = 0;
    for (int i=0; i<n; i++)
      sum += v1[i]*v2[i];
    return sum;
  }

  ~GQlambda() {
    delete [] theta;
    delete [] w;
    delete [] e;
  }

};