Introduction and Motivation
Here in this blog, we try to model Multi-step Time Series Prediction using Deep learning Models on the basis of Medical Information available for different states of India.
Multi-Step Time Series Prediction
Deep Learning-based Multi-variate Time Series Training and Prediction
- Feeding Multi-variate data from a single source or from aggregated sources available directly from the cloud or other 3rd-party providers into the ML modeling data ingestion system.
- Cleaning, preprocessing, and feature engineering of the data involving scaling and normalization.
- Conversion of the data to a supervised time-series.
- Feeding the data to a deep learning training source that can train different time-series models like LSTM, CNN, BI-LSTM, CNN+LSTM using different combinations of hidden layers, neurons, batch-size, and other hyper-parameters.
- Forecasting based on near term or far distant terms in the future either using Single-Step or Multi-Step Forecasting respectively.
- Evaluation of some of the error metrics like (MAPE, MAE, ME, RMSE, MPE) by comparing it with the actual data, when it comes in.
- Re-training the model and continuous improvements when the threshold of error exceeds.
Import Necessary Tensorflow Libraries
from tensorflow.python.keras.layers import Dense, LSTM, RepeatVector,TimeDistributed,Flatten, Bidirectional
from tensorflow.python.keras import Sequential
from tensorflow.python.keras.layers.convolutional import Conv1D, Conv2D, MaxPooling1D,ConvLSTM2D
Data Loading and Selecting Features
def split_dataset(data):
# split into standard weeks
print(np.shape(data))
split_factor = int((np.shape(data)[0]*0.7))
print("Split Factor no is", split_factor)
m = 7
trn_close_no = closestNumber(split_factor, m)
te_close_no = closestNumber((np.shape(data)[0]-split_factor), m)
train, test = data[0:trn_close_no], data[trn_close_no:(trn_close_no + te_close_no)]
print("Initials Train-Test Split --", np.shape(train), np.shape(test))
len_train = np.shape(train)[0]
len_test = np.shape(test)[0]
# restructure into windows of weekly data
train = array(split(train[0:len_train], len(train[0:len_train]) / 7))
test = array(split(test, len(test) / 7))
print("Final Train-Test Split --", np.shape(train), np.shape(test))
return train, test
scaler = MinMaxScaler(feature_range=(0, 1))
scaled_dataset = scaler.fit_transform(dataset)
Convert Time-Series to a Supervised DataSet
The below snippet code demonstrates what is described above.
# convert history into inputs and outputs
def to_supervised(train, n_input, n_out=7):
# flatten data
data = train.reshape((train.shape[0] * train.shape[1], train.shape[2]))
X, y = list(), list()
in_start = 0
# step over the entire history one time step at a time
for _ in range(len(data)):
# define the end of the input sequence
in_end = in_start + n_input
out_end = in_end + n_out
# ensure we have enough data for this instance
if out_end <= len(data):
X.append(data[in_start:in_end, :])
y.append(data[in_end:out_end, 0])
# move along one time step
in_start += 1
return array(X), array(y)
Training Different Deep Learning Models using Tensorflow
# train CNN model
def build_model_cnn(train, n_input):
# prepare data
train_x, train_y = to_supervised(train, n_input)
# define parameters
verbose, epochs, batch_size = 0, 200, 4
n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1]
# define model
model = Sequential()
model.add(Conv1D(filters=16, kernel_size=3, activation='relu', input_shape=(n_timesteps,n_features)))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(10, activation='relu'))
model.add(Dense(n_outputs))
model.compile(loss='mse', optimizer='adam')
# fit network
model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, verbose=verbose)
return model
LSTM
# train LSTM model
def build_model_lstm(train, n_input):
# prepare data
train_x, train_y = to_supervised(train, n_input)
print(np.shape(train_x))
print(np.shape(train_y))
# define parameters
verbose, epochs, batch_size = 0, 50, 16
n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1]
# reshape output into [samples, timesteps, features]
train_y = train_y.reshape((train_y.shape[0], train_y.shape[1], 1))
# define model
model = Sequential()
model.add(LSTM(200, activation='relu', input_shape=(n_timesteps, n_features)))
model.add(RepeatVector(n_outputs))
model.add(LSTM(200, activation='relu', return_sequences=True))
model.add(TimeDistributed(Dense(100, activation='relu')))
model.add(TimeDistributed(Dense(1)))
model.compile(loss='mse', optimizer='adam')
# fit network
model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, verbose=verbose)
return model
Bi-Directional LSTM
# train Bi-Directionsl LSTM model
def build_model_bi_lstm(train, n_input):
# prepare data
train_x, train_y = to_supervised(train, n_input)
print(np.shape(train_x))
print(np.shape(train_y))
# define parameters
verbose, epochs, batch_size = 0, 50, 16
n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1]
# reshape output into [samples, timesteps, features]
train_y = train_y.reshape((train_y.shape[0], train_y.shape[1], 1))
# define model
model = Sequential()
model.add(Bidirectional(LSTM(200, activation='relu', input_shape=(n_timesteps, n_features))))
model.add(RepeatVector(n_outputs))
model.add(Bidirectional(LSTM(200, activation='relu', return_sequences=True)))
model.add(TimeDistributed(Dense(100, activation='relu')))
model.add(TimeDistributed(Dense(1)))
model.compile(loss='mse', optimizer='adam')
# fit network
model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, verbose=verbose)
return model
Stacked LSTM + CNN
# train Stacked CNN + LSTM model
def build_model_cnn_lstm(train, n_input):
# prepare data
train_x, train_y = to_supervised(train, n_input)
# define parameters
verbose, epochs, batch_size = 0, 500, 16
n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1]
# reshape output into [samples, timesteps, features]
train_y = train_y.reshape((train_y.shape[0], train_y.shape[1], 1))
# define model
model = Sequential()
model.add(Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(n_timesteps, n_features)))
model.add(Conv1D(filters=64, kernel_size=3, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(RepeatVector(n_outputs))
model.add(LSTM(200, activation='relu', return_sequences=True))
model.add(TimeDistributed(Dense(100, activation='relu')))
model.add(TimeDistributed(Dense(1)))
model.compile(loss='mse', optimizer='adam')
# fit network
model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, verbose=verbose)
return model
Multi-Step Forecasting and Evaluation
# make a forecast
def forecast(model, history, n_input):
# flatten data
data = array(history)
data = data.reshape((data.shape[0]*data.shape[1], data.shape[2]))
# retrieve last observations for input data
input_x = data[-n_input:, :]
# reshape into [1, n_input, n]
input_x = input_x.reshape((1, input_x.shape[0], input_x.shape[1]))
# forecast the next week
yhat = model.predict(input_x, verbose=0)
# we only want the vector forecast
yhat = yhat[0]
return yhat
# evaluate one or more weekly forecasts against expected values
def evaluate_forecasts(actual, predicted):
print("Actual Results", np.shape(actual))
print("Predicted Results", np.shape(predicted))
scores = list()
# calculate an RMSE score for each day
for i in range(actual.shape[1]):
# calculate mse
mse = mean_squared_error(actual[:, i], predicted[:, i])
# calculate rmse
rmse = sqrt(mse)
# store
scores.append(rmse)
plt.figure(figsize=(14, 12))
plt.plot(actual[:, i], label='actual')
plt.plot(predicted[:, i], label='predicted')
plt.title(ModelType + ' based Multi-Step Time Series Active Cases Prediction for step ' + str(i))
plt.legend()
plt.show()
# calculate overall RMSE
s = 0
for row in range(actual.shape[0]):
for col in range(actual.shape[1]):
s += (actual[row, col] - predicted[row, col]) ** 2
score = sqrt(s / (actual.shape[0] * actual.shape[1]))
return score, scores
# evaluate a single model
def evaluate_model(train, test, n_input):
model = None
# fit model
if(ModelType == 'LSTM'):
print('lstm')
model = build_model_lstm(train, n_input)
elif(ModelType == 'BI_LSTM'):
print('bi_lstm')
model = build_model_bi_lstm(train, n_input)
elif(ModelType == 'CNN'):
print('cnn')
model = build_model_cnn(train, n_input)
elif(ModelType == 'LSTM_CNN'):
print('lstm_cnn')
model = build_model_cnn_lstm(train, n_input)
# history is a list of weekly data
history = [x for x in train]
# walk-forward validation over each week
predictions = list()
for i in range(len(test)):
# predict the week
yhat_sequence = forecast(model, history, n_input)
# store the predictions
predictions.append(yhat_sequence)
# get real observation and add to history for predicting the next week
history.append(test[i, :])
# evaluate predictions days for each week
predictions = array(predictions)
score, scores = evaluate_forecasts(test[:, :, 0], predictions)
return score, scores, test[:, :, 0], predictions
Multi-Step Conv2D + LSTM (Uni-variate & Multi-Variate) based Prediction for State Delhi
# train CONV LSTM2D model
def build_model_cnn_lstm_2d(train, n_steps, n_length, n_input):
# prepare data
train_x, train_y = to_supervised_2cnn_lstm(train, n_input)
# define parameters
verbose, epochs, batch_size = 0, 750, 16
n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1]
# reshape into subsequences [samples, time steps, rows, cols, channels]
train_x = train_x.reshape((train_x.shape[0], n_steps, 1, n_length, n_features))
# reshape output into [samples, timesteps, features]
train_y = train_y.reshape((train_y.shape[0], train_y.shape[1], 1))
# define model
model = Sequential()
model.add(ConvLSTM2D(filters=64, kernel_size=(1,3), activation='relu', input_shape=(n_steps, 1, n_length, n_features)))
model.add(Flatten())
model.add(RepeatVector(n_outputs))
model.add(LSTM(200, activation='relu', return_sequences=True))
model.add(TimeDistributed(Dense(100, activation='relu')))
model.add(TimeDistributed(Dense(1)))
model.compile(loss='mse', optimizer='adam')
# fit network
model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, verbose=verbose)
return model
# convert history into inputs and outputs
def to_supervised_2cnn_lstm(train, n_input, n_out=7):
# flatten data
data = train.reshape((train.shape[0]*train.shape[1], train.shape[2]))
X, y = list(), list()
in_start = 0
# step over the entire history one time step at a time
for _ in range(len(data)):
# define the end of the input sequence
in_end = in_start + n_input
out_end = in_end + n_out
# ensure we have enough data for this instance
if out_end <= len(data):
x_input = data[in_start:in_end, 0]
x_input = x_input.reshape((len(x_input), 1))
X.append(x_input)
y.append(data[in_end:out_end, 0])
# move along one time step
in_start += 1
return array(X), array(y)
# make a forecast
def forecast_2cnn_lstm(model, history, n_steps, n_length, n_input):
# flatten data
data = array(history)
data = data.reshape((data.shape[0]*data.shape[1], data.shape[2]))
# retrieve last observations for input data
input_x = data[-n_input:, 0]
# reshape into [samples, time steps, rows, cols, channels]
input_x = input_x.reshape((1, n_steps, 1, n_length, 1))
# forecast the next week
yhat = model.predict(input_x, verbose=0)
# we only want the vector forecast
yhat = yhat[0]
return yhat
# evaluate a single model
def evaluate_model_2cnn_lstm(train, test, n_steps, n_length, n_input):
# fit model
model = build_model_cnn_lstm_2d(train, n_steps, n_length, n_input)
# history is a list of weekly data
history = [x for x in train]
# walk-forward validation over each week
predictions = list()
for i in range(len(test)):
# predict the week
yhat_sequence = forecast_2cnn_lstm(model, history, n_steps, n_length, n_input)
# store the predictions
predictions.append(yhat_sequence)
# get real observation and add to history for predicting the next week
history.append(test[i, :])
# evaluate predictions days for each week
predictions = array(predictions)
score, scores = evaluate_forecasts(test[:, :, 0], predictions)
return score, scores, test[:, :, 0], predictions
df_state_all = pd.read_csv('all_states/all.csv')
df_state_all = df_state_all.drop(columns=['Latitude', 'Longitude', 'index'])
stateName = unique_states[8]
dataset = df_state_all[df_state_all['Name of State / UT'] == unique_states[8]]
dataset = dataset.sort_values(by='Date', ascending=True)
dataset = dataset[(dataset['Date'] >= '2020-03-25') & (dataset['Date'] <= '2020-06-06')]
print(np.shape(dataset))
daterange = dataset['Date'].values
no_Dates = len(daterange)
dateStart = daterange[0]
dateEnd = daterange[no_Dates - 1]
print(dateStart)
print(dateEnd)
dataset = dataset.drop(columns=['Unnamed: 0', 'Date', 'source1', 'state', 'Name of State / UT', 'tagpeopleinquarantine', 'tagtotaltested'])
print(np.shape(dataset))
n = np.shape(dataset)[0]
scaler = MinMaxScaler(feature_range=(0, 1))
scaled_dataset = scaler.fit_transform(dataset)
# split into train and test
train, test = split_dataset(scaled_dataset)
# define the number of subsequences and the length of subsequences
n_steps, n_length = 2, 7
# define the total days to use as input
n_input = n_length * n_steps
score, scores, actual, predicted = evaluate_model_2cnn_lstm(train, test, n_steps, n_length, n_input)
# summarize scores
summarize_scores(ModelType, score, scores)
#In function forecast_2cnn_lstm
input_x = data[-n_input:, :]. #replacing 0 with :
# reshape into [samples, time steps, rows, cols, channels]
input_x = input_x.reshape((1, n_steps, 1, n_length, data.shape[1])) #replacing 1 with #data.shape[1] for multi-variate
x_input = data[in_start:in_end, :]
x_input = x_input.reshape((len(x_input), x_input.shape[1]))
Conv2D + BI_LSTM
Conclusion
Acknowledgements
References
- https://arxiv.org/pdf/1801.02143.pdf
- https://machinelearningmastery.com/multi-step-time-series-forecasting/
- https://machinelearningmastery.com/multi-step-time-series-forecasting-with-machine-learning-models-for-household-electricity-consumption/
- https://machinelearningmastery.com/how-to-develop-lstm-models-for-multi-step-time-series-forecasting-of-household-power-consumption/
- https://machinelearningmastery.com/convert-time-series-supervised-learning-problem-python/
- https://www.tensorflow.org/tutorials/structured_data/time_series
- https://www.aiproblog.com/index.php/2018/11/13/how-to-develop-lstm-models-for-time-series-forecasting/