portfolio-manager.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import yfinance as yf\n",
    "import numpy as np\n",
    "import math\n",
    "from itertools import product, combinations\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from copy import copy\n",
    "from collections import deque\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PortfolioEnv():\n",
    "    def __init__(self, dates, datasets, n):\n",
    "        self.dates = dates\n",
    "        self.datasets = datasets\n",
    "        self.n = n\n",
    "        self.process_big_pt()\n",
    "        \n",
    "        self.process_small_pt()\n",
    "        \n",
    "        self.initial_pt = 1000000 # 1 000 000\n",
    "        self.c_minus = 0.0025 # 0.25%\n",
    "        self.c_plus = 0.0025 # 0.25%\n",
    "        self.delta = 10000 # 10 000\n",
    "                \n",
    "        self.process_actions()\n",
    "        self.action_shape = self.actions.shape[0]\n",
    "        self._episode_ended = False\n",
    "    \n",
    "    def reset(self):\n",
    "        # initialisation of portfolio\n",
    "        self.pt = self.initial_pt # 1 000 000\n",
    "        self.wt = [0.25,0.25,0.25,0.25] # wt before state 0\n",
    "        \n",
    "        self.current_tick = 0 # after checking, current_tick should be set to 0\n",
    "        self.episode_ended = False\n",
    "        \n",
    "        ktc = self.big_pt[self.current_tick,0,:,self.n-1] # nouveaux close prices des différents assets\n",
    "        wt_prime = (self.wt*self.phi(ktc)) / (np.dot(self.wt,self.phi(ktc))) # nouvelles proportions des assets du portfolio\n",
    "        \n",
    "        # after checking, we should return state 0\n",
    "        return {'big_xt':np.array(self.small_pt[self.current_tick]), 'wt_prime':wt_prime}\n",
    "    \n",
    "    def process_actions(self):\n",
    "        asset_number = 3\n",
    "        action_number = 3\n",
    "\n",
    "        seq = np.arange(asset_number)\n",
    "        actions = []\n",
    "\n",
    "        for c in product(seq, repeat=action_number):\n",
    "            actions.append(c)\n",
    "\n",
    "        self.actions = np.array(actions)\n",
    "    \n",
    "    def find_action_index(self,action):\n",
    "        for ind, a in enumerate(self.actions):\n",
    "            if np.array_equal(a, action):\n",
    "                return ind;\n",
    "    \n",
    "    def process_big_pt(self):\n",
    "        datasets = self.datasets\n",
    "        date_start = self.dates[0]\n",
    "        date_end = self.dates[1]\n",
    "\n",
    "        dfs = []\n",
    "        for d in datasets:\n",
    "            ticker = yf.Ticker(d)\n",
    "            # get historical market data\n",
    "            df_ = ticker.history(start=date_start, end=date_end, interval=\"1d\")    \n",
    "            df_.rename(mapper={\n",
    "                \"Close\": d+\"_close\",\n",
    "                \"Open\": d+\"_open\",\n",
    "                \"High\": d+\"_high\",\n",
    "                \"Low\": d+\"_low\",\n",
    "                \"Volume\": d+\"_volume\"\n",
    "            }, inplace=True, axis=1)\n",
    "            if \"Dividends\" in df_.columns:\n",
    "                df_.drop(axis=1,labels=[\"Dividends\", \"Stock Splits\"],inplace=True)\n",
    "            dfs.append(df_)\n",
    "\n",
    "        final_df = pd.concat(dfs, axis=1)\n",
    "        final_df.dropna(inplace=True)\n",
    "        \n",
    "        self.final_df = final_df\n",
    "        \n",
    "        final_df = self.final_df\n",
    "        n = self.n\n",
    "        Pc = []\n",
    "        for d in datasets:\n",
    "            asset_close = final_df[d+\"_close\"].values\n",
    "            asset_prev_close = final_df[d+\"_close\"].shift().values\n",
    "            Kc = (asset_close - asset_prev_close) / asset_prev_close\n",
    "            Kc = Kc[1:]\n",
    "            Pc_ = [Kc[i:i+n] for i in range(len(asset_close)-n)] # Kc[0:20], Kc[1:21], Kc[2:22]\n",
    "            Pc.append(Pc_)\n",
    "        Pc = np.array(Pc)\n",
    "        Po = []\n",
    "        for d in datasets:\n",
    "            asset_prev_close = final_df[d+\"_close\"].shift().values\n",
    "            asset_open = final_df[d+\"_open\"].values\n",
    "            Ko = (asset_open - asset_prev_close) / asset_prev_close\n",
    "            Ko = Ko[1:]\n",
    "            Po_ = [Ko[i:i+n] for i in range(len(asset_open)-n)]\n",
    "            Po.append(Po_)\n",
    "        Po = np.array(Po)\n",
    "        Pl = []\n",
    "        for d in datasets:\n",
    "            asset_close = final_df[d+\"_close\"].values\n",
    "            asset_low = final_df[d+\"_low\"].values\n",
    "            Kl = (asset_close - asset_low) / asset_low\n",
    "            Kl = Kl[1:]\n",
    "            Pl_ = [Kl[i:i+n] for i in range(len(asset_low)-n)]\n",
    "            Pl.append(Pl_)\n",
    "        Pl = np.array(Pl)\n",
    "        Ph = []\n",
    "        for d in datasets:\n",
    "            asset_close = final_df[d+\"_close\"].values\n",
    "            asset_high = final_df[d+\"_high\"].values\n",
    "            Kh = (asset_close - asset_high) / asset_high\n",
    "            Kh = Kh[1:]\n",
    "            Ph_ = [Kh[i:i+n] for i in range(len(asset_high)-n)]\n",
    "            Ph.append(Ph_)\n",
    "        Ph = np.array(Ph)\n",
    "        Pv = []\n",
    "        for d in datasets:\n",
    "            asset_prev_volume = final_df[d+\"_volume\"].shift().values\n",
    "            asset_volume = final_df[d+\"_volume\"].values\n",
    "            Kv = (asset_volume - asset_prev_volume) / asset_prev_volume\n",
    "            Kv = Kv[1:]\n",
    "            Pv_ = [Kv[i:i+n] for i in range(len(asset_high)-n)]\n",
    "            Pv.append(Pv_)\n",
    "        Pv = np.array(Pv)\n",
    "        Pt_star = np.array([Pc, Po, Pl, Ph, Pv])\n",
    "\n",
    "        self.big_pt = Pt_star.swapaxes(0,2).swapaxes(1,2)\n",
    "        print(self.big_pt.shape)\n",
    "        \n",
    "    def process_small_pt(self):\n",
    "        small_pt = []\n",
    "        \n",
    "        for big_xt in self.big_pt:\n",
    "            big_xt = big_xt.swapaxes(0,1).swapaxes(1,2)\n",
    "            big_xt_scaled = np.array([mm_scaler.fit_transform(a) for a in big_xt])\n",
    "\n",
    "            small_pt.append(big_xt_scaled)\n",
    "        \n",
    "        self.small_pt = np.array(small_pt)\n",
    "        print(self.small_pt.shape)\n",
    "    \n",
    "    def phi(self,v):\n",
    "        return np.insert(v, 0, [0]) + np.ones(len(v) + 1)\n",
    "    \n",
    "    def get_wt_prime_chapeau(self,wt_prime,big_s_minus,big_s_plus,pt_prime):\n",
    "        wt_prime_chapeau = []\n",
    "        for ind, key in enumerate(wt_prime):\n",
    "            if(ind in big_s_minus):\n",
    "                wt_prime_chapeau.append(key - self.delta/pt_prime)\n",
    "            elif(ind in big_s_plus):\n",
    "                wt_prime_chapeau.append(key + self.delta/pt_prime)\n",
    "            else:\n",
    "                wt_prime_chapeau.append(key)\n",
    "    \n",
    "        return np.array(wt_prime_chapeau)\n",
    "    \n",
    "    def is_asset_shortage(self,action,pt,wt):\n",
    "        big_s_minus = np.where(action==0)[0]\n",
    "        for ind in big_s_minus:\n",
    "            if wt[ind+1]*pt < self.delta:\n",
    "                return True\n",
    "        \n",
    "        return False\n",
    "    \n",
    "    #alright\n",
    "    def is_cash_shortage(self,action,pt,wt):\n",
    "        big_s_minus = np.where(action==0)[0]\n",
    "        big_s_plus = np.where(action==2)[0]\n",
    "        current_cash = wt[0]*pt\n",
    "        cash_after_selling = current_cash + (1-self.c_minus)*self.delta*len(big_s_minus) # must include transaction costs\n",
    "        cash_needed = (self.c_plus+1)*self.delta*len(big_s_plus) # must include transaction costs\n",
    "        \n",
    "        if(cash_after_selling < cash_needed):\n",
    "            return True\n",
    "        \n",
    "        return False\n",
    "    \n",
    "    def action_mapping(self,action,action_Q_values,pt,wt):\n",
    "        action = copy(action)\n",
    "        action_mapped = action\n",
    "        if self.is_asset_shortage(action,pt,wt):\n",
    "            action_mapped = self.rule2(action,action_Q_values,pt,wt)\n",
    "        elif self.is_cash_shortage(action,pt,wt):\n",
    "            action_mapped = self.rule1(action,action_Q_values,pt,wt)\n",
    "        \n",
    "        return action_mapped\n",
    "    \n",
    "    def rule1(self,action,action_Q_values,pt,wt):\n",
    "        MAXQ = np.NINF\n",
    "        action_selected = action\n",
    "        \n",
    "        big_s_plus = np.where(action==2)[0]\n",
    "        \n",
    "        for i in range(1,len(big_s_plus) + 1):\n",
    "            for c in combinations(big_s_plus, i):\n",
    "                new_action = copy(action)\n",
    "                for j in c:\n",
    "                    new_action[j] = 1\n",
    "\n",
    "                if not self.is_cash_shortage(new_action,pt,wt):\n",
    "                    new_action_index = self.find_action_index(new_action)\n",
    "                    new_action_Q_value = action_Q_values[new_action_index]\n",
    "                    \n",
    "                    if new_action_Q_value > MAXQ:\n",
    "                        MAXQ = new_action_Q_value\n",
    "                        action_selected = new_action\n",
    "        \n",
    "        return action_selected\n",
    "    \n",
    "    def rule2(self,action,action_Q_values,pt,wt):\n",
    "        for i in range(len(action)):\n",
    "            if wt[i+1]*pt < self.delta:\n",
    "                action[i] = 1\n",
    "        \n",
    "        if self.is_cash_shortage(action,pt,wt):\n",
    "            action = self.rule1(action,action_Q_values,pt,wt)\n",
    "        \n",
    "        return action\n",
    "    \n",
    "    def F(self,pt,wt):\n",
    "        action_possible = []\n",
    "        for ind, action in enumerate(self.actions):\n",
    "            if not self.is_asset_shortage(action,pt,wt) and not self.is_cash_shortage(action,pt,wt):\n",
    "                action_possible.append(ind)\n",
    "\n",
    "        return np.array(action_possible)\n",
    "    \n",
    "    def step(self, action, simulation=False):\n",
    "        # Must set new portfolio with regards to action\n",
    "        # Must set new reward\n",
    "        if self.current_tick == len(self.big_pt) - 2:\n",
    "            # The last action ended the episode. Ignore the current action and start\n",
    "            # a new episode.\n",
    "            self.episode_ended = True\n",
    "        \n",
    "        # we are in state 0, best action between state 0 and state 1 has been predicted\n",
    "        # so we must get portfolio value and weights after state 0 evolution\n",
    "        # but before action has been taken into account\n",
    "        ktc = self.big_pt[self.current_tick,0,:,self.n-1] # nouveaux close prices des différents assets\n",
    "        pt_prime = self.pt * np.dot(self.wt,self.phi(ktc)) # nouvelle valeur du portfolio issue de l'action précédente\n",
    "        wt_prime = (self.wt*self.phi(ktc)) / (np.dot(self.wt,self.phi(ktc))) # nouvelles proportions des assets du portfolio\n",
    "        \n",
    "        # On prend en compte la nouvelle action\n",
    "        big_s_minus = np.where(action==0)[0]\n",
    "        big_s_plus = np.where(action==2)[0]\n",
    "        \n",
    "        ct = (self.delta*(self.c_minus*len(big_s_minus) + self.c_plus*len(big_s_plus)))/pt_prime\n",
    "        if not simulation:\n",
    "            self.pt = pt_prime*(1 - ct)\n",
    "        else:\n",
    "            pt = pt_prime*(1 - ct)\n",
    "        \n",
    "        wt_prime_chapeau_1tillend = self.get_wt_prime_chapeau(wt_prime[1:],big_s_minus,big_s_plus,pt_prime)\n",
    "        wt_prime_chapeau_0 = wt_prime[0] + self.delta*((1-self.c_minus)*len(big_s_minus)-(1+self.c_plus)*len(big_s_plus))/pt_prime\n",
    "        wt_prime_chapeau = np.concatenate((np.array([wt_prime_chapeau_0]), wt_prime_chapeau_1tillend))\n",
    "        \n",
    "        # now we evolve to state one, to get reward of this action\n",
    "        if not simulation:\n",
    "            self.wt = wt_prime_chapeau / (np.dot(wt_prime_chapeau, np.ones(len(wt_prime_chapeau))))\n",
    "            self.current_tick += 1\n",
    "            k_t_plus_one_c = self.big_pt[self.current_tick,0,:,self.n-1]\n",
    "        else:\n",
    "            wt = wt_prime_chapeau / (np.dot(wt_prime_chapeau, np.ones(len(wt_prime_chapeau))))\n",
    "            current_tick = self.current_tick + 1\n",
    "            k_t_plus_one_c = self.big_pt[current_tick,0,:,self.n-1]\n",
    "        \n",
    "        big_p_s_t_plus_one = pt_prime*np.dot(wt_prime, self.phi(k_t_plus_one_c))\n",
    "        \n",
    "        if not simulation:\n",
    "            p_t_plus_one_prime = self.pt * np.dot(self.wt,self.phi(k_t_plus_one_c))\n",
    "            reward = (p_t_plus_one_prime - big_p_s_t_plus_one)/big_p_s_t_plus_one\n",
    "            wt_plus_one_prime = (self.wt*self.phi(k_t_plus_one_c)) / (np.dot(self.wt,self.phi(k_t_plus_one_c)))\n",
    "        else:\n",
    "            p_t_plus_one_prime = pt * np.dot(wt,self.phi(k_t_plus_one_c))\n",
    "            reward = (p_t_plus_one_prime - big_p_s_t_plus_one)/big_p_s_t_plus_one\n",
    "            wt_plus_one_prime = (wt*self.phi(k_t_plus_one_c)) / (np.dot(wt,self.phi(k_t_plus_one_c)))\n",
    "        \n",
    "        if not simulation:\n",
    "            return {'big_xt':np.array(self.small_pt[self.current_tick]), 'wt_prime':wt_plus_one_prime}, reward, self.episode_ended\n",
    "        else:\n",
    "            return {'big_xt':np.array(self.small_pt[current_tick]), 'wt_prime':wt_plus_one_prime}, reward, self.episode_ended, p_t_plus_one_prime, wt_plus_one_prime\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3000, 5, 3, 20)\n",
      "(3000, 3, 20, 5)\n"
     ]
    }
   ],
   "source": [
    "mm_scaler = MinMaxScaler()\n",
    "\n",
    "datasets = [\"SPY\", \"IWD\", \"IWC\"]\n",
    "train_dates = [\"2006-01-01\", \"2017-12-30\"] # let's train it on more datas to see what happens\n",
    "n = 20\n",
    "env = PortfolioEnv(train_dates, datasets, n)\n",
    "datas_ = env.big_pt\n",
    "datas = datas_.swapaxes(2,3).swapaxes(1,3)\n",
    "final_datas = []\n",
    "for d in datas:\n",
    "    final_datas.append(d[0])\n",
    "    final_datas.append(d[1])\n",
    "    final_datas.append(d[2])\n",
    "final_datas = np.array(final_datas)\n",
    "datas_scaled = np.array([mm_scaler.fit_transform(d) for d in final_datas]) # MinMaxScaler par ligne ou par colonne ?\n",
    "X_train = datas_scaled[:7200]\n",
    "X_valid = datas_scaled[7200:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm (LSTM)                  (None, 20, 128)           68608     \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 20)                11920     \n",
      "=================================================================\n",
      "Total params: 80,528\n",
      "Trainable params: 80,528\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "repeat_vector (RepeatVector) (None, 20, 20)            0         \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 20, 20)            3280      \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 20, 128)           76288     \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 20, 5)             645       \n",
      "=================================================================\n",
      "Total params: 80,213\n",
      "Trainable params: 80,213\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "sequential (Sequential)      (None, 20)                80528     \n",
      "_________________________________________________________________\n",
      "sequential_1 (Sequential)    (None, 20, 5)             80213     \n",
      "=================================================================\n",
      "Total params: 160,741\n",
      "Trainable params: 160,741\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/50\n",
      "225/225 [==============================] - 3s 13ms/step - loss: 0.6523 - val_loss: 0.6408\n",
      "Epoch 2/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6411 - val_loss: 0.6343\n",
      "Epoch 3/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6357 - val_loss: 0.6289\n",
      "Epoch 4/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6314 - val_loss: 0.6259\n",
      "Epoch 5/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6284 - val_loss: 0.6238\n",
      "Epoch 6/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6267 - val_loss: 0.6219\n",
      "Epoch 7/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6252 - val_loss: 0.6204\n",
      "Epoch 8/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6237 - val_loss: 0.6200\n",
      "Epoch 9/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6226 - val_loss: 0.6191\n",
      "Epoch 10/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6216 - val_loss: 0.6177\n",
      "Epoch 11/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6194 - val_loss: 0.6162\n",
      "Epoch 12/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6174 - val_loss: 0.6148\n",
      "Epoch 13/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6159 - val_loss: 0.6142\n",
      "Epoch 14/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6137 - val_loss: 0.6123\n",
      "Epoch 15/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6115 - val_loss: 0.6113\n",
      "Epoch 16/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6093 - val_loss: 0.6103\n",
      "Epoch 17/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6071 - val_loss: 0.6087\n",
      "Epoch 18/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6054 - val_loss: 0.6071\n",
      "Epoch 19/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6037 - val_loss: 0.6074\n",
      "Epoch 20/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6019 - val_loss: 0.6058\n",
      "Epoch 21/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.6011 - val_loss: 0.6042\n",
      "Epoch 22/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5992 - val_loss: 0.6038\n",
      "Epoch 23/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5979 - val_loss: 0.6019\n",
      "Epoch 24/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5957 - val_loss: 0.6023\n",
      "Epoch 25/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5948 - val_loss: 0.6003\n",
      "Epoch 26/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5935 - val_loss: 0.5992\n",
      "Epoch 27/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5921 - val_loss: 0.5990\n",
      "Epoch 28/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5908 - val_loss: 0.5988\n",
      "Epoch 29/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5901 - val_loss: 0.5983\n",
      "Epoch 30/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5884 - val_loss: 0.5984\n",
      "Epoch 31/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5876 - val_loss: 0.5981\n",
      "Epoch 32/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5872 - val_loss: 0.5972\n",
      "Epoch 33/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5857 - val_loss: 0.5973\n",
      "Epoch 34/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5847 - val_loss: 0.5978\n",
      "Epoch 35/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5830 - val_loss: 0.5971\n",
      "Epoch 36/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5826 - val_loss: 0.5978\n",
      "Epoch 37/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5827 - val_loss: 0.5963\n",
      "Epoch 38/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5815 - val_loss: 0.5963\n",
      "Epoch 39/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5795 - val_loss: 0.5962\n",
      "Epoch 40/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5793 - val_loss: 0.5956\n",
      "Epoch 41/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5776 - val_loss: 0.5954\n",
      "Epoch 42/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5769 - val_loss: 0.5949\n",
      "Epoch 43/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5763 - val_loss: 0.5948\n",
      "Epoch 44/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5749 - val_loss: 0.5952\n",
      "Epoch 45/50\n",
      "225/225 [==============================] - 2s 8ms/step - loss: 0.5748 - val_loss: 0.5959\n",
      "Epoch 46/50\n",
      "225/225 [==============================] - 2s 9ms/step - loss: 0.5737 - val_loss: 0.5951\n",
      "Epoch 47/50\n",
      "225/225 [==============================] - 2s 9ms/step - loss: 0.5734 - val_loss: 0.5960\n",
      "Epoch 48/50\n",
      "225/225 [==============================] - 2s 9ms/step - loss: 0.5725 - val_loss: 0.5953\n",
      "Epoch 49/50\n",
      "225/225 [==============================] - 2s 9ms/step - loss: 0.5733 - val_loss: 0.5962\n",
      "Epoch 50/50\n",
      "225/225 [==============================] - 2s 9ms/step - loss: 0.5712 - val_loss: 0.5959\n"
     ]
    }
   ],
   "source": [
    "recurrent_encoder = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.LSTM(units=128, input_shape=[20,5], return_sequences=True),\n",
    "    tf.keras.layers.LSTM(units=20),\n",
    "])\n",
    "recurrent_decoder = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.RepeatVector(20, input_shape=[20]),\n",
    "    tf.keras.layers.LSTM(units=20, return_sequences=True),\n",
    "    tf.keras.layers.LSTM(units=128, return_sequences=True),\n",
    "    tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(5, activation=\"sigmoid\"))\n",
    "])\n",
    "recurrent_ae = tf.keras.models.Sequential([recurrent_encoder, recurrent_decoder])\n",
    "recurrent_ae.compile(optimizer='adam', loss='binary_crossentropy')\n",
    "recurrent_encoder.summary()\n",
    "recurrent_decoder.summary()\n",
    "recurrent_ae.summary()\n",
    "\n",
    "history = recurrent_ae.fit(X_train, X_train, epochs=50, validation_data=(X_valid, X_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(230, 5, 3, 20)\n",
      "(230, 3, 20, 5)\n",
      "(231, 5, 3, 20)\n",
      "(231, 3, 20, 5)\n",
      "(229, 5, 3, 20)\n",
      "(229, 3, 20, 5)\n",
      "(230, 5, 3, 20)\n",
      "(230, 3, 20, 5)\n",
      "(230, 5, 3, 20)\n",
      "(230, 3, 20, 5)\n",
      "(230, 5, 3, 20)\n",
      "(230, 3, 20, 5)\n",
      "(231, 5, 3, 20)\n",
      "(231, 3, 20, 5)\n"
     ]
    }
   ],
   "source": [
    "def create_envs(dates, datasets, n):\n",
    "    return np.array([PortfolioEnv(d, datasets, n) for d in dates])\n",
    "datasets = [\"SPY\", \"IWD\", \"IWC\"]\n",
    "train_dates = [\n",
    "    [\"2010-01-01\", \"2010-12-30\"],\n",
    "    [\"2011-01-01\", \"2011-12-30\"],\n",
    "    [\"2012-01-01\", \"2012-12-30\"],\n",
    "    [\"2013-01-01\", \"2013-12-30\"],\n",
    "    [\"2014-01-01\", \"2014-12-30\"],\n",
    "    [\"2015-01-01\", \"2015-12-30\"],\n",
    "    [\"2016-01-01\", \"2016-12-30\"],\n",
    "    [\"2017-01-01\", \"2017-12-30\"],\n",
    "]\n",
    "train_envs = create_envs(train_dates, datasets, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(231, 5, 3, 20)\n",
      "(231, 3, 20, 5)\n"
     ]
    }
   ],
   "source": [
    "test_dates = [\n",
    "    [\"2018-01-01\", \"2018-12-30\"],\n",
    "]\n",
    "test_envs = create_envs(test_dates, datasets, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            [(None, 20, 5)]      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_3 (InputLayer)            [(None, 20, 5)]      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_4 (InputLayer)            [(None, 20, 5)]      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sequential_3 (Sequential)       (None, 20)           80528       input_2[0][0]                    \n",
      "                                                                 input_3[0][0]                    \n",
      "                                                                 input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            [(None, 4)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 60)           0           sequential_3[56][0]              \n",
      "                                                                 sequential_3[57][0]              \n",
      "                                                                 sequential_3[58][0]              \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 64)           0           input_1[0][0]                    \n",
      "                                                                 concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 64)           4160        concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 32)           2080        dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 27)           891         dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 87,659\n",
      "Trainable params: 87,659\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef2617c7dfe7407286fa94ef053ca45a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=229.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1000000\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d15ee40b73b4a758e6535c35de6b293",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=229.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1000000\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b21995f69a8647dca747972ba5c0134f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=229.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1000000\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1af5c6f699c4adf9e88ab281d0704dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=229.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1000000\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbd44f42aab14271bbcac4c984bdbc76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=229.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1000000\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74015f5bf9e64de8b18654d6c1d75a5e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=229.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1000000\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73ae4c3a4e5d484c9b31e0b7e3ce2deb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=228.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1000000\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ee03dacf7f944e4aa220ab89f8ca089",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=229.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1000000\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc28b90f1df54652b17410221281782a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=230.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1000000\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6786144e4a44d919bfa173ee54259e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=230.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1000000\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ec31fb7a6114012973ca1b8b77d80ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=230.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1000000\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95f2cd4e129846b3813abae34ce452e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=230.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1000000\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62f1324b54d44f7bbcc1c4117a71a3df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=230.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1000000\n",
      "1000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d657fda02f314d1e8b8038c0ef3ec7ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=229.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-58-51223501ec3e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    113\u001b[0m                 \u001b[0mpreprocessed_states\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwts_prime\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrewards\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreprocessed_next_states\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdones\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnext_pts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnext_wts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msimulation\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfield_index\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msimulation\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msimulations\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfield_index\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 115\u001b[0;31m                 \u001b[0mnext_Q_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext_wts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreprocessed_next_states\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreprocessed_next_states\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreprocessed_next_states\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Q-values predicted for all next_states\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    116\u001b[0m                 \u001b[0mmax_next_Q_values_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext_Q_values\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# for each next_states find index of max Q-value predicted by target\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     86\u001b[0m       raise ValueError('{} is not supported in multi-worker mode.'.format(\n\u001b[1;32m     87\u001b[0m           method.__name__))\n\u001b[0;32m---> 88\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m   return tf_decorator.make_decorator(\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose, steps, callbacks, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1283\u001b[0m             \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_predict_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'outputs'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_outputs\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1284\u001b[0m       \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_predict_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1285\u001b[0;31m     \u001b[0mall_outputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_structure_up_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconcat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1286\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mtf_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_numpy_or_python_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_outputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1287\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36mmap_structure_up_to\u001b[0;34m(shallow_tree, func, *inputs, **kwargs)\u001b[0m\n\u001b[1;32m   1116\u001b[0m       \u001b[0;32mlambda\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;31m# Discards the path arg.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1117\u001b[0m       \u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1118\u001b[0;31m       **kwargs)\n\u001b[0m\u001b[1;32m   1119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36mmap_structure_with_tuple_paths_up_to\u001b[0;34m(shallow_tree, func, *inputs, **kwargs)\u001b[0m\n\u001b[1;32m   1212\u001b[0m                     in _yield_flat_up_to(shallow_tree, inputs[0], is_seq)]\n\u001b[1;32m   1213\u001b[0m   results = [func(*args, **kwargs) for args in zip(flat_path_list,\n\u001b[0;32m-> 1214\u001b[0;31m                                                    *flat_value_lists)]\n\u001b[0m\u001b[1;32m   1215\u001b[0m   return pack_sequence_as(structure=shallow_tree, flat_sequence=results,\n\u001b[1;32m   1216\u001b[0m                           expand_composites=expand_composites)\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   1211\u001b[0m   flat_path_list = [path for path, _\n\u001b[1;32m   1212\u001b[0m                     in _yield_flat_up_to(shallow_tree, inputs[0], is_seq)]\n\u001b[0;32m-> 1213\u001b[0;31m   results = [func(*args, **kwargs) for args in zip(flat_path_list,\n\u001b[0m\u001b[1;32m   1214\u001b[0m                                                    *flat_value_lists)]\n\u001b[1;32m   1215\u001b[0m   return pack_sequence_as(structure=shallow_tree, flat_sequence=results,\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(_, *values)\u001b[0m\n\u001b[1;32m   1114\u001b[0m   return map_structure_with_tuple_paths_up_to(\n\u001b[1;32m   1115\u001b[0m       \u001b[0mshallow_tree\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1116\u001b[0;31m       \u001b[0;32mlambda\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;31m# Discards the path arg.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1117\u001b[0m       \u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1118\u001b[0m       **kwargs)\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mconcat\u001b[0;34m(tensors, axis)\u001b[0m\n\u001b[1;32m   1738\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mragged_tensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRaggedTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1739\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mragged_concat_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1740\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1741\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1742\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    178\u001b[0m     \u001b[0;34m\"\"\"Call target, and fall back on dispatchers if there is a TypeError.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 180\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    181\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mTypeError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m       \u001b[0;31m# Note: convert_to_eager_tensor currently raises a ValueError, not a\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py\u001b[0m in \u001b[0;36mconcat\u001b[0;34m(values, axis, name)\u001b[0m\n\u001b[1;32m   1602\u001b[0m       ops.convert_to_tensor(\n\u001b[1;32m   1603\u001b[0m           \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"concat_dim\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1604\u001b[0;31m           dtype=dtypes.int32).get_shape().assert_has_rank(0)\n\u001b[0m\u001b[1;32m   1605\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0midentity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1606\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mgen_array_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat_v2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import time\n",
    "import pickle as pkl\n",
    "\n",
    "@tf.function\n",
    "def train_step(preprocessed_states, wts_prime, mask, target_Q_values):\n",
    "    with tf.device('gpu:0'):\n",
    "        with tf.GradientTape() as tape:\n",
    "            all_Q_values = model((wts_prime, preprocessed_states[:,0], preprocessed_states[:,1], preprocessed_states[:,2]))\n",
    "            Q_values = tf.reduce_sum(all_Q_values * mask, axis=1, keepdims=True)\n",
    "            loss = tf.reduce_mean(loss_fn(target_Q_values, Q_values))\n",
    "            # Q_values ne concerne pas les actions pas concernées, de même que target_Q_values\n",
    "        grads = tape.gradient(loss, model.trainable_variables)\n",
    "        optimizer.apply_gradients(zip(grads, model.trainable_variables))\n",
    "\n",
    "beta = .3\n",
    "epochs = 50 \n",
    "\n",
    "r = np.random.rand(epochs)\n",
    "N=len(train_envs)\n",
    "gen_trunc=(N-1-np.floor(np.log(1-r*(1-(1-beta)**N))/np.log(1-beta))).astype(int)\n",
    "\n",
    "tf.keras.backend.clear_session()\n",
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "K = tf.keras.backend\n",
    "\n",
    "wt_prime_inpt = tf.keras.layers.Input(shape=[4])\n",
    "inputs = [wt_prime_inpt]\n",
    "encoders = []\n",
    "for t in datasets:\n",
    "    inpt = tf.keras.layers.Input(shape=[20, 5])\n",
    "    inputs.append(inpt)\n",
    "    encoder = recurrent_encoder(inpt)\n",
    "    encoders.append(encoder)\n",
    "\n",
    "enc_concat = tf.keras.layers.Concatenate(axis=1)(encoders)\n",
    "wt_prime_concat_with_enc = tf.keras.layers.Concatenate(axis=1)([wt_prime_inpt, enc_concat])\n",
    "\n",
    "hidden1 = tf.keras.layers.Dense(64, activation=\"relu\")(wt_prime_concat_with_enc)\n",
    "hidden2 = tf.keras.layers.Dense(32, activation=\"relu\")(hidden1)\n",
    "output = tf.keras.layers.Dense(train_envs[0].action_shape)(hidden2)\n",
    "\n",
    "model = tf.keras.models.Model(inputs=[inputs], outputs=[output])\n",
    "model.summary()\n",
    "\n",
    "target = tf.keras.models.clone_model(model)\n",
    "target.set_weights(model.get_weights())\n",
    "\n",
    "replay_memory = deque(maxlen=2000)\n",
    "\n",
    "epsilon = 1\n",
    "batch_size = 32\n",
    "discount_rate = .9\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(lr=1e-7)\n",
    "loss_fn = tf.keras.losses.mean_squared_error\n",
    "\n",
    "steps = 0\n",
    "episode_counter = 1\n",
    "\n",
    "train_CR = [[] for env in train_envs]\n",
    "previous_CR = 0\n",
    "\n",
    "test_CR = []\n",
    "test_previous_CR = 0\n",
    "\n",
    "for ind_env in gen_trunc:\n",
    "    env = train_envs[ind_env]\n",
    "    state = env.reset()\n",
    "        \n",
    "    pbar = tqdm(total=len(env.big_pt)-1)\n",
    "        \n",
    "    while 1: # will break when end of env\n",
    "        pbar.set_description(f'e.{episode_counter}/{len(gen_trunc)}|tr.p.CR:{previous_CR}|te.p.CR:{test_previous_CR}')\n",
    "        #epsilon-greedy policy to select an action\n",
    "        action = None        \n",
    "\n",
    "        k_t_plus_one_c = env.big_pt[env.current_tick,0,:,env.n-1]\n",
    "        p_t_plus_one_prime = env.pt * np.dot(env.wt,env.phi(k_t_plus_one_c))\n",
    "        wt_plus_one_prime = (env.wt*env.phi(k_t_plus_one_c)) / (np.dot(env.wt,env.phi(k_t_plus_one_c)))\n",
    "        \n",
    "        if np.random.rand() < epsilon: # goal is to take uniformly random action among possible actions of this env\n",
    "            possible_actions = env.F(p_t_plus_one_prime, wt_plus_one_prime) # when we sell/buy it's new prices, not protfolio\n",
    "            # value when action A has been taken, so when action A has been taken and then new state has come\n",
    "\n",
    "            random_index = np.random.randint(len(possible_actions))\n",
    "            action = env.actions[possible_actions[random_index]]\n",
    "        else: # here goal is to make model predict best action, but if action isn't feasable for env, map it\n",
    "            Q_values = model.predict((state['wt_prime'][np.newaxis], state['big_xt'][0][np.newaxis], state['big_xt'][1][np.newaxis], state['big_xt'][2][np.newaxis]))\n",
    "            action = env.action_mapping(env.actions[np.argmax(Q_values[0])], Q_values[0], p_t_plus_one_prime, wt_plus_one_prime)\n",
    "        \n",
    "        # simulate all feasible actions for current state\n",
    "        possible_actions = env.F(p_t_plus_one_prime, wt_plus_one_prime)\n",
    "        simulations = []\n",
    "        \n",
    "        for simulated_action in possible_actions:\n",
    "            next_state_simulated, next_reward, done, next_pt, next_wt = env.step(env.actions[simulated_action], simulation=True)\n",
    "\n",
    "            simulations.append((state['big_xt'], state['wt_prime'], simulated_action, next_reward, next_state_simulated['big_xt'], done, next_pt, np.array(next_wt)))\n",
    "        replay_memory.append(np.array(simulations))\n",
    "        next_state, reward, episode_ended = env.step(action)\n",
    "        # do some stuff with it\n",
    "        \n",
    "        # add some time for buffer to populate\n",
    "        \n",
    "        if steps >= 100:\n",
    "            # start batch training (room to improvment, why don't use Prioritized Experience Replay ?)\n",
    "            # select a uniformly random batch\n",
    "            indices = np.random.randint(len(replay_memory), size=batch_size)\n",
    "            batch = [replay_memory[index] for index in indices]\n",
    "            for simulations in batch:\n",
    "                #start = time.time()\n",
    "                preprocessed_states, wts_prime, actions, rewards, preprocessed_next_states, dones, next_pts, next_wts = [np.array([simulation[field_index] for simulation in simulations]) for field_index in range(8)]\n",
    "                \n",
    "                next_Q_values = target.predict((next_wts, preprocessed_next_states[:,0], preprocessed_next_states[:,1], preprocessed_next_states[:,2])) # Q-values predicted for all next_states\n",
    "                max_next_Q_values_index = np.argmax(next_Q_values, axis=1) # for each next_states find index of max Q-value predicted by target\n",
    "                \n",
    "                # okay, here, in max_next_Q_values, there are some actions that may not be feasible and that we need to map\n",
    "                # but we have to use is_asset_shortage, is_cash_shortage and action_mapping with state of simulation\n",
    "                max_next_Q_values_actions = np.array([\n",
    "                    env.actions[action_simulated] if not env.is_asset_shortage(env.actions[action_simulated], next_pts[ind], next_wts[ind]) and not env.is_cash_shortage(env.actions[action_simulated], next_pts[ind], next_wts[ind])\n",
    "                    else env.action_mapping(env.actions[action_simulated], next_Q_values[ind], next_pts[ind], next_wts[ind])\n",
    "                    for ind, action_simulated in enumerate(max_next_Q_values_index)\n",
    "                ])\n",
    "                \n",
    "                # okay, we have the feasible actions selected by the target network, now we need to find their indexes to communicate with the network\n",
    "                max_next_Q_values_indexes = [env.find_action_index(action_simulated) for action_simulated in max_next_Q_values_actions]\n",
    "                # now we have to find the Q-values predicted by target network of these corrected actions\n",
    "                max_next_Q_values = [next_Q_values[ind][best_action_simulated] for ind, best_action_simulated in enumerate(max_next_Q_values_indexes)]\n",
    "\n",
    "                # and now, we can proprely calculate the target Q-value\n",
    "                target_Q_values = (rewards +\n",
    "                                   (1 - dones) * discount_rate * max_next_Q_values) # here problem with dones /!\\\n",
    "                target_Q_values = target_Q_values.reshape(-1, 1)\n",
    "                mask = tf.one_hot(actions, env.action_shape)\n",
    "                \n",
    "                train_step(preprocessed_states, wts_prime, mask, target_Q_values)\n",
    "                # and we're done, amazing !!\n",
    "                #print('updates ::: {:.5f} ms / step'.format((time.time() - start) * 1000))\n",
    "            \n",
    "        state = next_state\n",
    "\n",
    "        steps += 1\n",
    "        pbar.update(1)\n",
    "        \n",
    "        if episode_ended:\n",
    "            break;\n",
    "        \n",
    "    episode_counter += 1\n",
    "    \n",
    "    previous_CR = round((env.pt-env.initial_pt)/env.initial_pt, 3)\n",
    "    train_CR[ind_env].append(previous_CR)\n",
    "    pbar.close()\n",
    "    \n",
    "    for test_env in test_envs:\n",
    "        test_state = test_env.reset()\n",
    "        # state 0 return\n",
    "        # but current_tick set to one\n",
    "        test_episode_ended = False\n",
    "        while not test_episode_ended:\n",
    "            # state 0\n",
    "            # Q-values of action from state 0 predicted\n",
    "            # algorithm only knows state 0 and want to predict best action and then state one happens\n",
    "            \n",
    "            test_Q_values = model.predict((test_state['wt_prime'][np.newaxis], test_state['big_xt'][0][np.newaxis], test_state['big_xt'][1][np.newaxis], test_state['big_xt'][2][np.newaxis]))\n",
    "            \n",
    "            # we want to get total value of portfolio and new portfolio weights regarding state 0 evolution\n",
    "            # so we enter the algorithm with an already predefined portfolio distribution\n",
    "            # then state 0 happens\n",
    "            # so here current_tick should 0, not one,\n",
    "            # to calculate total value of portfolio and new portfolio weights regarding state 0 evolution\n",
    "            test_k_t_plus_one_c = test_env.big_pt[test_env.current_tick,0,:,test_env.n-1]\n",
    "            test_p_t_plus_one_prime = test_env.pt * np.dot(test_env.wt,test_env.phi(test_k_t_plus_one_c))\n",
    "            test_wt_plus_one_prime = (test_env.wt*test_env.phi(test_k_t_plus_one_c)) / (np.dot(test_env.wt,test_env.phi(test_k_t_plus_one_c)))\n",
    "            \n",
    "            test_action = test_env.action_mapping(test_env.actions[np.argmax(test_Q_values[0])], test_Q_values[0], test_p_t_plus_one_prime, test_wt_plus_one_prime)\n",
    "            \n",
    "            # okay here, we have selected best action predicted after state 0 evolution\n",
    "            # and we want to evolve to state one\n",
    "            test_state, test_reward, test_episode_ended = test_env.step(test_action)\n",
    "        \n",
    "        test_previous_CR = round((test_env.pt-test_env.initial_pt)/test_env.initial_pt, 3)\n",
    "        test_CR.append(test_previous_CR)\n",
    "        \n",
    "    # update of target network\n",
    "    target.set_weights(model.get_weights()) \n",
    "    epsilon = (1-episode_counter/500) # max(1 - episode_counter / 400, 0.01)\n",
    "    \n",
    "    if episode_counter % 25 == 0:\n",
    "        checkpoint_path = \"training_2/cp-{epoch:04d}.ckpt\"\n",
    "        model.save_weights(checkpoint_path.format(epoch=episode_counter))\n",
    "        pkl.dump(train_CR, open('train_CR.pkl', 'wb'))\n",
    "        pkl.dump(test_CR, open('test_CR.pkl', 'wb'))\n",
    "        \n",
    "# In[ ]:\n",
    "pkl.dump(train_CR, open('train_CR.pkl', 'wb'))\n",
    "pkl.dump(test_CR, open('test_CR.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "name": "tf2-2-2-gpu.2-2.m50",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-2-2-gpu.2-2:m50"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}