MLA_Learn_Periodic_Functions/code/wilshire_5000/wilshire.py

import numpy as np
import tensorflow as tf
import pandas as pd
import matplotlib.pyplot as plt


def parser(path):
    df = pd.read_csv(path,na_values='.')
    print(df.shape)
    #df = df.interpolate()
    df = df.dropna().reset_index(drop=True)
    print(df.shape)
    #df = df.drop(labels=np.arange(1825)) ### To obtain the same graph than in the article
    print(df.shape)
    return(df)

def preprocess(path):
    df = parser(path)
    
    df_normalized = df[:]
    df_normalized["WILL5000INDFC"]=df_normalized["WILL5000INDFC"]/np.max(df_normalized["WILL5000INDFC"])
    index_train = int(df_normalized[df_normalized["DATE"]=="2020-01-31"].index.array[0])
    print(df)
    print(df_normalized)
    print(df_normalized[df_normalized["DATE"]=="2020-01-31"])
    # df.plot()
    # plt.show()
    df_train = df_normalized[:index_train]
    df_test = df_normalized[index_train+1:index_train+85]

    # df_train.plot()
    # df_test.plot()
    # plt.show()
    return(df_train,df_test,index_train)