vérification de l'apprentissage du wilshire5000 et ajout de la météo par des RNN

code/RNN_prediction_meteo/.gitignore

+data

code/RNN_prediction_meteo/README.md

+# Prédictions météo
+
+## Données
+
+[Données](https://donneespubliques.meteofrance.fr/?fond=produit&id_produit=90&id_rubrique=32)

code/RNN_prediction_meteo/grab_data.py

+import os
+import pandas
+import numpy as np
+import progressbar
+from matplotlib import pyplot as plt
+
+station = 7460 # clermont ferand <3
+downlaod = True
+
+if downlaod :
+	print("Suppresion des fichiers existants")
+	os.system("rm -rf data")
+	os.system("mkdir data")
+	print("Début du téléchargement depuis météo france")
+	for annee in progressbar.progressbar(range(1996,2021)) :
+		for moi in range(1,13) :
+			titre = f'{annee:04d}{moi:02d}'
+			os.system(f'wget -q https://donneespubliques.meteofrance.fr/donnees_libres/Txt/Synop/Archive/synop.{titre}.csv.gz -P data')
+
+	os.chdir("data")
+	os.system(f"gzip -vd *")
+	os.chdir("..")
+
+
+donnees = {}
+for nom in os.listdir("data") :
+	csv = pandas.read_csv(f"data/{nom}",sep=";")
+	extraction = csv[csv["numer_sta"]==station][["date","t"]].values.tolist()
+
+	for i in range(len(extraction)) :
+		mesure = int(extraction[i][0]//1e6//7)
+		if mesure not in donnees :
+			donnees[mesure] = []
+		try :
+			donnees[mesure].append(float(extraction[i][1]))
+		except ValueError :
+			print(f"Erreur de valeur pour la date {extraction[i][0]} : {extraction[i][1]}")
+
+
+clefs = list(donnees.keys())
+clefs.sort()
+table = np.zeros(len(clefs))
+for i, clef in enumerate(clefs) :
+	somme = 0
+	for j in range(len(donnees[clef])) :
+		somme += donnees[clef][j]
+	if len(donnees[clef]) > 0 :
+		table[i] = somme/len(donnees[clef])
+	else :
+		table[i] = table[i-1]
+
+plt.plot(np.array(range(len(table)))/52-len(table)/52,table,"bo")
+np.save("dataset.npy", table)
+print("sauvegardé")
+plt.show()
+
+
+

code/RNN/training.py → code/RNN_prediction_meteo/training.py

 
 LEN_SEQ = 64
 PRED = 0.05
-HIDDEN = 128
+HIDDEN = 32
 
 model = MyModel(HIDDEN)
 
 fig, axs = plt.subplots(2)
 axs[0].plot(annee[:start_pred],dataset[:start_pred], label="apprentissage")
 axs[0].plot(annee[start_pred:],dataset[start_pred:], label="validation")
-axs[1].plot(annee, data_Pred, label="prediction")
+axs[1].plot(annee[:start_pred],dataset[:start_pred], label="données")
+axs[1].plot(annee[start_pred:], data_Pred[start_pred:], label="prediction")
 plt.legend()
 plt.show()

code/RNN_wilshire_5000/Model.py

+import tensorflow as tf
+
+class MyModel(tf.keras.Model):
+
+	def __init__(self, HIDDEN):
+		super(MyModel, self).__init__()
+		self.lstm1 = tf.keras.layers.LSTM(HIDDEN, return_sequences=True)
+		#self.lstm2 = tf.keras.layers.LSTM(HIDDEN, return_sequences=True)
+		#self.lstm3 = tf.keras.layers.LSTM(HIDDEN, return_sequences=True)
+		#self.lstm4 = tf.keras.layers.LSTM(HIDDEN, return_sequences=True)
+		self.lstmlast = tf.keras.layers.LSTM(HIDDEN, return_sequences=True)
+		self.dense1 = tf.keras.layers.Dense(HIDDEN, activation='relu')
+		self.dense2 = tf.keras.layers.Dense(HIDDEN//2, activation='relu')
+		self.denselast = tf.keras.layers.Dense(1, activation='sigmoid')
+
+
+	def call(self, inputs):
+		x = self.lstm1(inputs)
+		#x = self.lstm2(x)
+		#x = self.lstm3(x)
+		#x = self.lstm4(x)
+		x = self.lstmlast(x)
+		x = self.dense1(x)
+		x = self.dense2(x)
+		x = self.denselast(x)
+		return x

code/RNN_wilshire_5000/training.py

+import tensorflow as tf
+from matplotlib import pyplot as plt
+import numpy as np
+from Model import MyModel
+
+
+LEN_SEQ = 64
+PRED = 0.027
+HIDDEN = 128
+
+model = MyModel(HIDDEN)
+
+dataset = np.load('dataset.npy')
+scale = (np.max(dataset) - np.min(dataset))
+data = dataset/scale
+shift = np.min(data)
+data = data - shift
+
+annee = np.array(list(range(len(data))))/365
+annee = annee - annee[-1]
+start_pred = int(len(data)*(1-PRED))
+print(len(data))
+print(start_pred)
+plt.figure(1)
+plt.plot(annee[:start_pred],data[:start_pred], label="apprentissage")
+plt.plot(annee[start_pred:],data[start_pred:], label="validation")
+plt.legend()
+plt.show()
+
+X_train_tot = [data[0:start_pred-1]]
+Y_train_tot = [data[1:start_pred]]
+
+X_train_tot = np.expand_dims(np.array(X_train_tot),2)
+Y_train_tot = np.expand_dims(np.array(Y_train_tot),2)
+
+X_train = X_train_tot[:,:LEN_SEQ,:]
+Y_train = Y_train_tot[:,:LEN_SEQ,:]
+
+for i in range(len(X_train_tot[0]) - LEN_SEQ) :
+	X_train = np.concatenate((X_train, X_train_tot[:,i:i+LEN_SEQ,:]),0)
+	Y_train = np.concatenate((Y_train, Y_train_tot[:,i:i+LEN_SEQ,:]),0)
+print(X_train_tot.shape)
+print(X_train.shape)
+
+
+model.compile(optimizer='adam',
+			  loss='binary_crossentropy',
+			  metrics=['binary_crossentropy'])
+import os 
+os.system("rm -rf log_dir")
+
+model.fit(x=X_train, y=Y_train, batch_size=16, epochs=5, shuffle=True)
+
+Pred = X_train_tot.copy()
+while len(Pred[0]) < len(data) :
+	print(len(data) - len(Pred[0]))
+	Pred = np.concatenate((Pred, np.array([[model.predict(Pred)[0][-1]]])),1)
+Pred = Pred + shift
+Pred = Pred * scale
+
+data_Pred = np.squeeze(Pred)
+
+plt.figure(2)
+plt.plot(annee[:start_pred],dataset[:start_pred], label="apprentissage")
+plt.plot(annee[start_pred:],dataset[start_pred:], label="validation")
+plt.plot(annee, data_Pred, label="prediction")
+plt.legend()
+
+fig, axs = plt.subplots(2)
+axs[0].plot(annee[:start_pred],dataset[:start_pred], label="données")
+axs[0].plot(annee[start_pred:],dataset[start_pred:], label="validation")
+axs[1].plot(annee[:start_pred],dataset[:start_pred], label="données")
+axs[1].plot(annee[start_pred:], data_Pred[start_pred:], label="prediction")
+plt.legend()
+plt.show()