[agent_keras_rl.py] ValueError: Error when checking : expected reshape_1_input to have 6 dimensions, but got array with shape (1, 1, 51, 10, 2)
dragon28 opened this issue · comments
Hello Mr. Marc Juchli,
Good day.
When I execute python3 agent_keras_rl.py
agent_keras_rl.py source code:
import logging
import numpy as np
from rl.agents.dqn import DQNAgent
from rl.policy import EpsGreedyQPolicy
from rl.memory import SequentialMemory
from ctc_executioner.order_side import OrderSide
from ctc_executioner.orderbook import Orderbook
from ctc_executioner.agent_utils.action_plot_callback import ActionPlotCallback
from ctc_executioner.agent_utils.live_plot_callback import LivePlotCallback
from keras.models import Sequential
from keras.layers import Dense, Activation, Flatten, LSTM, Reshape
from keras.optimizers import Adam, SGD
from keras import regularizers
from keras import optimizers
from collections import deque
import gym
#logging.basicConfig(level=logging.INFO)
from rl.callbacks import Callback
class EpsDecayCallback(Callback):
def __init__(self, eps_poilcy, decay_rate=0.95):
self.eps_poilcy = eps_poilcy
self.decay_rate = decay_rate
def on_episode_begin(self, episode, logs={}):
self.eps_poilcy.eps *= self.decay_rate
print('eps = %s' % self.eps_poilcy.eps)
def createModel():
# Neural Net for Deep-Q learning Model
model = Sequential()
model.add(Reshape((env.observation_space.shape[0], env.observation_space.shape[1]*2), input_shape=(1, 1)+env.observation_space.shape))
#model.add(Flatten(input_shape=(env.observation_space.shape[0], env.observation_space.shape[1], env.observation_space.shape[2])))
model.add(LSTM(512, activation='tanh', recurrent_activation='tanh'))
#model.add(Dense(4*env.bookSize*env.lookback))
#model.add(Dense(env.bookSize*env.lookback))#, kernel_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l1(0.01)))
#model.add(Dense(4*env.bookSize))
#model.add(Activation('relu'))
#model.add(Flatten())
model.add(Dense(len(env.levels)))
model.add(Activation('linear'))
#model.compile(optimizers.SGD(lr=.1), "mae")
model.summary()
return model
def loadModel(name):
# load json and create model
from keras.models import model_from_json
json_file = open(name + '.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json)
model.load_weights(name + '.h5')
print('Loaded model "' + name + '" from disk')
return model
def saveModel(model, name):
# serialize model to JSON
model_json = model.to_json()
with open(name + '.json', "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
model.save_weights(name + '.h5')
print('Saved model "' + name + '" to disk')
# # Load orderbook
orderbook = Orderbook()
orderbook.loadFromEvents('data/events/ob-train.tsv')
orderbook_test = orderbook
orderbook.summary()
import datetime
orderbook = Orderbook()
config = {
'startPrice': 10000.0,
# 'endPrice': 9940.0,
'priceFunction': lambda p0, s, samples: p0 + 10 * np.sin(2*np.pi*10 * (s/samples)),
'levels': 50,
'qtyPosition': 0.1,
'startTime': datetime.datetime.now(),
'duration': datetime.timedelta(seconds=1000),
'interval': datetime.timedelta(seconds=1)
}
orderbook.createArtificial(config)
orderbook.summary()
#orderbook.plot(show_bidask=True)
import gym_ctc_executioner
env = gym.make("ctc-executioner-v0")
import gym_ctc_marketmaker
#env = gym.make("ctc-marketmaker-v0")
env.setOrderbook(orderbook)
#model = loadModel(name='model-sell-artificial-2')
#model = loadModel(name='model-sell-artificial-sine')
model = createModel()
nrTrain = 100000
nrTest = 10
policy = EpsGreedyQPolicy()
memory = SequentialMemory(limit=5000, window_length=1)
# nb_steps_warmup: the default value for that in the DQN OpenAI baselines implementation is 1000
dqn = DQNAgent(model=model, nb_actions=len(env.levels), memory=memory, nb_steps_warmup=100, target_model_update=1e-2, policy=policy)
dqn.compile(Adam(lr=1e-3), metrics=['mae'])
cbs_train = []
cbs_train = [LivePlotCallback(nb_episodes=20000, avgwindow=20)]
dqn.fit(env, nb_steps=nrTrain, visualize=True, verbose=2, callbacks=cbs_train)
saveModel(model=model, name='model-sell-artificial-sine')
#cbs_train = []
#cbs_test = []
#cbs_test = [ActionPlotCallback(nb_episodes=nrTest)]
#dqn.test(env, nb_episodes=nrTest, visualize=True, verbose=2, callbacks=cbs_test)
Which lead to the following error message:
python3 agent_keras_rl.py
Using TensorFlow backend.
Attempt to load from cache.
Order book in cache. Load...
Number of states: 4883
Duration: 5510.176
States per second: 0.8861785903027416
Change of price per second: 4.8450654909561335
Number of states: 1001
Duration: 1000.0
States per second: 1.001
Change of price per second: 0.39930722266504565
[2019-05-28 14:28:51,734] Making new env: ctc-executioner-v0
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
reshape_1 (Reshape) (None, 51, 20) 0
_________________________________________________________________
lstm_1 (LSTM) (None, 512) 1091584
_________________________________________________________________
dense_1 (Dense) (None, 101) 51813
_________________________________________________________________
activation_1 (Activation) (None, 101) 0
=================================================================
Total params: 1,143,397
Trainable params: 1,143,397
Non-trainable params: 0
_________________________________________________________________
2019-05-28 14:28:52.824047: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX
Training for 100000 steps ...
Traceback (most recent call last):
File "agent_keras_rl.py", line 114, in <module>
dqn.fit(env, nb_steps=nrTrain, visualize=True, verbose=2, callbacks=cbs_train)
File "/home/dragon/quant/python/ctc-executioner/venv/lib/python3.6/site-packages/rl/core.py", line 160, in fit
action = self.forward(observation)
File "/home/dragon/quant/python/ctc-executioner/venv/lib/python3.6/site-packages/rl/agents/dqn.py", line 217, in forward
q_values = self.compute_q_values(state)
File "/home/dragon/quant/python/ctc-executioner/venv/lib/python3.6/site-packages/rl/agents/dqn.py", line 69, in compute_q_values
q_values = self.compute_batch_q_values([state]).flatten()
File "/home/dragon/quant/python/ctc-executioner/venv/lib/python3.6/site-packages/rl/agents/dqn.py", line 64, in compute_batch_q_values
q_values = self.model.predict_on_batch(batch)
File "/home/dragon/quant/python/ctc-executioner/venv/lib/python3.6/site-packages/keras/models.py", line 1039, in predict_on_batch
return self.model.predict_on_batch(x)
File "/home/dragon/quant/python/ctc-executioner/venv/lib/python3.6/site-packages/keras/engine/training.py", line 1946, in predict_on_batch
self._feed_input_shapes)
File "/home/dragon/quant/python/ctc-executioner/venv/lib/python3.6/site-packages/keras/engine/training.py", line 113, in _standardize_input_data
'with shape ' + str(data_shape))
ValueError: Error when checking : expected reshape_1_input to have 6 dimensions, but got array with shape (1, 1, 51, 10, 2)
I am using Ubuntu 18.04.2 x64 and Python version 3.6.7
Thanks
Hello
manage to fixed it by changing
model.add(Reshape((env.observation_space.shape[0], env.observation_space.shape[1]*2), input_shape=(1, 1)+env.observation_space.shape)
to
model.add(Reshape((env.observation_space.shape[0], env.observation_space.shape[1]*2), input_shape=(1,)+env.observation_space.shape))
Thanks
Great! Sorry, have been a bit busy lately, but do not hesitate to open an issue if you have another question.
@dragon28 did you use an external orderbook, or did you use the same as the data here?