#install yfinance in Colab
!pip install yfinance

#import the required libraries
import pandas as pd
import numpy as np
import keras
import tensorflow as tf
import plotly.graph_objects as go
from keras.preprocessing.sequence import TimeseriesGenerator
import math
import pandas_datareader as web
import numpy as np
import pandas as pd
from datetime import datetime
from sklearn.preprocessing import MinMaxScaler
import matplotlib.pyplot as plt
from pandas_datareader import data as pdr
import yfinance as yfin
from keras.models import Sequential
from keras.layers import LSTM, Dense, Dropout
import plotly.graph_objects as go
from sklearn.preprocessing import StandardScaler
from tensorflow.keras.optimizers import Adam
from plotly.io import write_html
import datetime as dt
from datetime import datetime
from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, TensorBoard
import model_helper

#yfinance api call to import data
yfin.pdr_override()
df = pdr.get_data_yahoo("^GSPC ^VIX", start="2002-01-01", end="2022-05-03")

#data cleanup
df["sp500"] = df["Adj Close"]["^GSPC"]
df["volume"] = df["Volume"]["^GSPC"]
df["vix"] = df["Adj Close"]["^VIX"]
df = df.reset_index()
df = df.drop(columns = ["Adj Close", "Volume"])
df = df.drop(columns = ["Close", "High", "Low", "Open"])
df.head()

#replace with datetime
df['Date'] = pd.to_datetime(df['Date'])
df.set_axis(df['Date'], inplace=True)

#visualize our data
trace1 = go.Scatter(
    x = df["Date"],
    y = df["sp500"],
    mode = 'lines',
    name = 'SP500'
)
layout = go.Layout(
    title = "S&P500 Index from 2002 to 2022",
    xaxis = {'title' : "Date"},
    yaxis = {'title' : "Close"}
)
fig = go.Figure(data=[trace1], layout=layout)
write_html(fig, "sp500.html")

#creates dataset for training and testing purposes
close_data = df['sp500'].values
close_data = close_data.reshape((-1,1))

split_percent = 0.80
split = int(split_percent*len(close_data))

#training and testing
close_train = close_data[:split]
close_test = close_data[split:]

date_train = df['Date'][:split]
date_test = df['Date'][split:]

#this will be the length for our RNN input
look_back = 20

#generate time series using this function
train_generator = TimeseriesGenerator(close_train, close_train, length=look_back, batch_size=20)     
test_generator = TimeseriesGenerator(close_test, close_test, length=look_back, batch_size=1)

#make our first model
model = Sequential()
#LSTM layer is the backbone of our RNN
model.add(
    LSTM(50,
        return_sequences=True, 
        activation='relu',
        input_shape=(look_back,1))
)
model.add(
    LSTM(50, 
        activation='relu',
        input_shape=(look_back,1))
)
#dropout layer to prevent overfitting
model.add(Dropout(0.2))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse')

#fit our data with our model
model.fit_generator(train_generator, epochs=25, verbose=1)

#make prediction
prediction = model.predict_generator(test_generator)

#reshape our prediction
close_train = close_train.reshape((-1))
close_test = close_test.reshape((-1))
prediction = prediction.reshape((-1))

#plots the three segments of data points, the training data, the predicted trend, and the actual price
trace1 = go.Scatter(
    x = date_train,
    y = close_train,
    mode = 'lines',
    name = 'Training Data'
)
trace2 = go.Scatter(
    x = date_test,
    y = prediction,
    mode = 'lines',
    name = 'Prediction'
)
trace3 = go.Scatter(
    x = date_test,
    y = close_test,
    mode='lines',
    name = 'Actual Price'
)
layout = go.Layout(
    title = "SP500 Prediction",
    xaxis = {'title' : "Date"},
    yaxis = {'title' : "Close"}
)
fig = go.Figure(data=[trace1, trace2, trace3], layout=layout)
write_html(fig, "prediction1.html")

#reshape our data
close_data = close_data.reshape((-1))

num_prediction = 60
forecast = predict(num_prediction, model)
forecast_dates = predict_dates(num_prediction)

#imports the actual price for these dates and cleans up
result = pdr.get_data_yahoo("^GSPC ^VIX", start="2022-05-02", end="2022-06-03")
result["sp500"] = result["Adj Close"]["^GSPC"]
result["voulme"] = result["Volume"]["^GSPC"]
result["vix"] = result["Adj Close"]["^VIX"]
result = result.reset_index()
result = result.drop(columns = ["Adj Close", "Volume"])
result = result.drop(columns = ["Close", "High", "Low", "Open"])
result['Date'] = pd.to_datetime(result['Date'])
result.set_axis(result['Date'], inplace=True)
close_data = result['sp500'].values
close_data = close_data.reshape((-1,1))
actual_date = result['Date']
actual_close = close_data

#plots the real-world prediction
trace1 = go.Scatter(
    x = forecast_dates,
    y = forecast,
    mode = 'lines',
    name = 'Data'
)
trace2 = go.Scatter(
    x = result['Date'],
    y = result['sp500'].values,
    mode = 'lines',
    name = 'Actual Price'
)
layout = go.Layout(
    title = "SP500 Prediction",
    xaxis = {'title' : "Date"},
    yaxis = {'title' : "Close"}
)
fig = go.Figure(data=[trace1, trace2], layout=layout)

#Data import and cleanup
dataset_train = pd.read_csv('^NDX.csv')
cols = list(dataset_train)[1:]
datelist_train = list(dataset_train["Date"])
datelist_train = [dt.datetime.strptime(date, '%Y-%m-%d').date() for date in datelist_train]
training_set = dataset_train.values

#feature scaling using StandardScalar
sc = StandardScaler()
training_set_scaled = sc.fit_transform(training_set)
#shapes our data
sc_predict = StandardScaler()
sc_predict.fit_transform(training_set[:, 0:1])

#create our training data set
X_train = []
y_train = []

#predict 60 days with 90 days
n_future = 60   
n_past = 90     

for i in range(n_past, len(training_set_scaled) - n_future +1):
    X_train.append(training_set_scaled[i - n_past:i, 0:dataset_train.shape[1] - 1])
    y_train.append(training_set_scaled[i + n_future - 1:i + n_future, 0])

X_train, y_train = np.array(X_train), np.array(y_train)

#Similar deal, sequential model with 2 LSTM models and one dropout
model = Sequential()
model.add(LSTM(units=64, 
               return_sequences=True, 
               input_shape=(n_past, dataset_train.shape[1]-1)))
model.add(LSTM(units=10, 
               return_sequences=False))
model.add(Dropout(0.20))
model.add(Dense(units=1, activation='linear'))

#compile the model
model.compile(optimizer = Adam(learning_rate=0.01), loss='mean_squared_error')

#fit the model
model.fit(X_train, y_train, epochs=20)

#makes the list for prediction
datelist_future = pd.date_range(datelist_train[-1], periods=n_future, freq='1d').tolist()

#makes the list of dates
datelist_future_ = []
for this_timestamp in datelist_future:
    datelist_future_.append(this_timestamp.date())

#makes the prediction
predictions_future = model.predict(X_train[-n_future:])
predictions_train = model.predict(X_train[n_past:])

#cleans up the data for visualization
y_pred_future = sc_predict.inverse_transform(predictions_future)
y_pred_train = sc_predict.inverse_transform(predictions_train)

PREDICTIONS_FUTURE = pd.DataFrame(y_pred_future, columns=['Open']).set_index(pd.Series(datelist_future))
PREDICTION_TRAIN = pd.DataFrame(y_pred_train, columns=['Open']).set_index(pd.Series(datelist_train[2 * n_past + n_future -1:]))
PREDICTION_TRAIN.index = PREDICTION_TRAIN.index.to_series()
dataset_train = pd.DataFrame(dataset_train, columns=cols)
dataset_train.index = datelist_train
dataset_train.index = pd.to_datetime(dataset_train.index)

START_DATE_FOR_PLOTTING = '2012-05-01'

trace1 = go.Scatter(
    x = PREDICTIONS_FUTURE.index,
    y = PREDICTIONS_FUTURE['Open'],
    mode = 'lines',
    name = 'Predicted Stock Price'
)
trace2 = go.Scatter(
    x = PREDICTION_TRAIN.loc[START_DATE_FOR_PLOTTING:].index,
    y = PREDICTION_TRAIN.loc[START_DATE_FOR_PLOTTING:]['Open'],
    mode = 'lines',
    name = 'Training Predictions'
)
trace3 = go.Scatter(
    x = dataset_train.loc[START_DATE_FOR_PLOTTING:].index,
    y = dataset_train.loc[START_DATE_FOR_PLOTTING:]['Open'],
    mode = 'lines',
    name = 'Actual Stock Prices'
)
layout = go.Layout(
    title = "NQ100 Prediction",
    xaxis = {'title' : "Date"},
    yaxis = {'title' : "Close"}
)
fig = go.Figure(data=[trace1, trace2, trace3], layout=layout)
write_html(fig, "prediction3.html")

yfin.pdr_override()
df = pdr.get_data_yahoo("^GSPC ^VIX", start="2020-03-15", end="2022-06-05")
df = df.reset_index()

df.head()
dataset_train = pd.concat([df['Date'],df['Open']['^GSPC'],df['High']['^GSPC'],df['Low']['^GSPC'],df['Close']['^GSPC'],df['Adj Close']['^GSPC'],df['Volume']['^GSPC']],axis=1)
dataset_train.columns=["Date","Open","High","Low","Close","Adj Close","Volume"]

cols = list(dataset_train)[1:]
datelist_train = list(dataset_train['Date'])
datelist_train = [dt.datetime.strptime(str(date), '%Y-%m-%d %H:%M:%S').date() for date in datelist_train]

training_set = dataset_train[cols].astype(str)
training_set = training_set.astype(float)

training_set = training_set.values
training_set

sc = StandardScaler()
training_set_scaled = sc.fit_transform(training_set)

sc_predict = StandardScaler()
sc_predict.fit_transform(training_set[:, 0:1])

# Creating a data structure with 90 timestamps and 1 output
X_train = []
y_train = []

n_future = 60  
n_past = 90     

for i in range(n_past, len(training_set_scaled) - n_future +1):
    X_train.append(training_set_scaled[i - n_past:i, 0:dataset_train.shape[1] - 1])
    y_train.append(training_set_scaled[i + n_future - 1:i + n_future, 0])

X_train, y_train = np.array(X_train), np.array(y_train)

print('X_train shape == {}.'.format(X_train.shape))
print('y_train shape == {}.'.format(y_train.shape))

model = Sequential()

model.add(LSTM(units=64, return_sequences=True, input_shape=(n_past, dataset_train.shape[1]-1)))

model.add(LSTM(units=10, return_sequences=False))

model.add(Dropout(0.25))

model.add(Dense(units=1, activation='linear'))

model.compile(optimizer = Adam(learning_rate=0.01), loss='mean_squared_error')
rlr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=10, verbose=1)
mcp = ModelCheckpoint(filepath='weights.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True)

tb = TensorBoard('logs')

history = model.fit(X_train, y_train, shuffle=True, epochs=30, callbacks=[rlr, mcp, tb], validation_split=0.2, verbose=1, batch_size=256)

datelist_future = pd.date_range(datelist_train[-1], periods=n_future, freq='1d').tolist()
datelist_future_ = []
for this_timestamp in datelist_future:
    datelist_future_.append(this_timestamp.date())

predictions_future = model.predict(X_train[-n_future:])

predictions_train = model.predict(X_train[n_past:])

y_pred_future = sc_predict.inverse_transform(predictions_future)
y_pred_train = sc_predict.inverse_transform(predictions_train)

PREDICTIONS_FUTURE = pd.DataFrame(y_pred_future, columns=['Open']).set_index(pd.Series(datelist_future))
PREDICTION_TRAIN = pd.DataFrame(y_pred_train, columns=['Open']).set_index(pd.Series(datelist_train[2 * n_past + n_future -1:]))

# Convert <datetime.date> to <Timestamp> for PREDCITION_TRAIN
PREDICTION_TRAIN.index = PREDICTION_TRAIN.index.to_series().apply(datetime_to_timestamp)

PREDICTION_TRAIN

trace1 = go.Scatter(
    x = PREDICTIONS_FUTURE.index,
    y = PREDICTIONS_FUTURE['Open'],
    mode = 'lines',
    name = 'Predicted Stock Price'
)
trace2 = go.Scatter(
    x = PREDICTION_TRAIN.index,
    y = PREDICTION_TRAIN['Open'],
    mode = 'lines',
    name = 'Training Predictions'
)
trace3 = go.Scatter(
    x = dataset_train['Date'],
    y = dataset_train['Open'],
    mode = 'lines',
    name = 'Actual Stock Prices'
)
layout = go.Layout(
    title = "NQ100 Prediction",
    xaxis = {'title' : "Date"},
    yaxis = {'title' : "Close"}
)
fig = go.Figure(data=[trace1, trace2, trace3], layout=layout)
fig.show()
write_html(fig, "prediction4.html")