main.cpp

#include <algorithm>
#include <array>
#include <iostream>
#include <random>
#include <thread>

using std::string;

using QTable = std::array<std::array<double, 2>, 21>;

std::random_device rd;
std::mt19937 gen(rd());
//std::uniform_int_distribution<> computer_choice_dist(0, 9);
//computerChoice = computer_choice_dist(gen);


enum class Action {
    LEFT = 0,
    RIGHT = 1
};

class Environment {
private:
    const std::pair<int, int> range;
    const int initial_position;
    int position;


public:
    void reset() {
        position = initial_position;
    }

    int step(const Action &action) {
        switch (action) {
        case Action::LEFT:
            position -= 1;
            break;
        case Action::RIGHT:
            position += 1;
            break;
        }

        position = std::clamp(position, range.first, range.second);

        return position;
    }

    Environment(const std::pair<int, int>& range, const int& position)
        : range(range), initial_position(position), position(position) {}

    int getPosition() const { return position; }

};
class QLearning {
private:
    QTable qTable{};
    const double ALPHA = 0.8; // Learning rate
    const double GAMMA = 0.9; // Discount factor
    const double EPSILON = 0.2; // Exploration rate


public:
    double getQValue(const int& position, const Action& action) const {
        return qTable[position + 10][static_cast<int>(action)];
    };

    Action chooseAction(const int& position, const bool& explore = true) const {
        std::uniform_real_distribution<> exploration(0, 1);
        std::uniform_int_distribution<> random_action(0, 1);
        if (explore && exploration(gen) < EPSILON) {
            return (random_action(gen) == 0) ? Action::LEFT : Action::RIGHT; // Random action
        }
        // Choose the action with the highest Q-value
        const double leftQ = getQValue(position, Action::LEFT);
        const double rightQ = getQValue(position, Action::RIGHT);
        return (leftQ > rightQ) ? Action::LEFT : Action::RIGHT;
    }

    void learn(const int& position, const Action& action, const double& reward, const int& nextPosition) {
        const double currentQ = getQValue(position, action);
        const double nextMaxQ = std::max(getQValue(nextPosition, Action::LEFT), getQValue(nextPosition, Action::RIGHT));
        const double newQ = currentQ + ALPHA * (reward + GAMMA * nextMaxQ - currentQ);
        qTable[position + 10][static_cast<int>(action)] = newQ;

    }

    QTable getQTable() const { return qTable; }
};

int bestPath(const int& initial_position, const QLearning& qLearning, const int& goal, const std::pair<int, int>& range) {
    Environment env(range, initial_position);
    int current_position = initial_position;
    bool done = false;
    int steps = 0;

    while (!done && steps < 2000) {
        Action action = qLearning.chooseAction(current_position, false);
        const int nextPosition = env.step(action);
        steps++;
        if (nextPosition == goal) {
            done = true;
        } else {
            current_position = nextPosition;
        }
    }

    return done ? steps : -1;
}

void printQTable(const QTable& table) {
    for (int i = 0; i < table.size(); ++i) {
        std::cout << "Position: " << i - 10 << ", Left Q-Value: "
                  << table[i][static_cast<int>(Action::LEFT)] << ", Right Q-Value: "
                  << table[i][static_cast<int>(Action::RIGHT)] << "\n";
    }
}

int main() {
    constexpr int GOAL = 5;
    constexpr int EPISODES = 1000;
    QLearning qLearning;

    constexpr std::pair<int, int> range = {-10, 10};


    for (int i = 0; i < EPISODES; i++) {
        std::uniform_int_distribution<> random_position(range.first, range.second);
        int current_position = random_position(gen);
        Environment env(range, current_position);
        std::cout << "Starting at position " << current_position << " ";
        bool done = false;
        int steps = 0;
        while (!done) {
            const Action action = qLearning.chooseAction(env.getPosition());
            const int nextPosition = env.step(action);
            const double reward = (nextPosition == GOAL) ? 1 : -0.1;
            done = (nextPosition == GOAL);
            qLearning.learn(current_position, action, reward, nextPosition);
            current_position = nextPosition;
            steps++;
        }
        std::cout << "Episode " << i + 1 << " completed in " << steps << " steps.\n";
    }


    std::cout << "\n\n Q-Table after training:\n";
    printQTable(qLearning.getQTable());

    std::cout << "\n\n Best Path:\n";

    for (int i = range.first; i <= range.second; i++) {
        int steps = bestPath(i, qLearning, GOAL, range);
        if (steps != -1) {
            std::cout << "Starting at position " << i << " took " << steps << " steps to reach the goal.\n";
        } else {
            std::cout << "Starting at position " << i << " did not reach the goal.\n";
        }
    }

    return 0;

}