model2_NRM.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"accelerator":"GPU","colab":{"name":"Model2_NRM.ipynb","provenance":[],"collapsed_sections":[],"machine_shape":"hm","authorship_tag":"ABX9TyPc7M7UDxSdY2BcvKVypQHi"},"kernelspec":{"display_name":"Python 3","name":"python3"}},"cells":[{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"qDKMog461J_l","executionInfo":{"status":"ok","timestamp":1607937725841,"user_tz":360,"elapsed":663,"user":{"displayName":"Zr Xiang","photoUrl":"","userId":"10347344196072452629"}},"outputId":"73aaacc2-1af8-45d3-bb4b-a85f854bf52f"},"source":["from google.colab import drive\r\n","drive.mount('/content/drive')"],"execution_count":24,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"u631zTBG1MFo","executionInfo":{"status":"ok","timestamp":1607937726403,"user_tz":360,"elapsed":1178,"user":{"displayName":"Zr Xiang","photoUrl":"","userId":"10347344196072452629"}},"outputId":"06faf431-b446-4d45-fea9-aa95125ada5d"},"source":["%cd '/content/drive/My Drive/ModelSharing'"],"execution_count":25,"outputs":[{"output_type":"stream","text":["/content/drive/My Drive/ModelSharing\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"gPPqYh3U1VL4","executionInfo":{"status":"ok","timestamp":1607937726405,"user_tz":360,"elapsed":1156,"user":{"displayName":"Zr Xiang","photoUrl":"","userId":"10347344196072452629"}}},"source":["import numpy as np\r\n","import pandas as pd\r\n","\r\n","from tensorflow.keras.models import *\r\n","from tensorflow.keras.layers import *\r\n","from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\r\n","from tensorflow.keras.optimizers import *\r\n","import tensorflow.keras.backend as K\r\n","\r\n","from sklearn.preprocessing import MinMaxScaler\r\n","from sklearn.model_selection import train_test_split"],"execution_count":26,"outputs":[]},{"cell_type":"code","metadata":{"id":"GGyMOPlH1ihI","executionInfo":{"status":"ok","timestamp":1607937726406,"user_tz":360,"elapsed":1132,"user":{"displayName":"Zr Xiang","photoUrl":"","userId":"10347344196072452629"}}},"source":["def series_to_supervised(data, n_in=1, n_out=1, dropnan=False):\r\n","  n_vars = 1 if type(data) is list else data.shape[1]\r\n","  df = pd.DataFrame(data)\r\n","  cols, names = list(), list()\r\n","  # input sequence (t-n, ... t-1)\r\n","  for i in range(n_in, 0, -1):\r\n","    cols.append(df.shift(i))\r\n","    names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]\r\n","  # forecast sequence (t, t+1, ... t+n)\r\n","  for i in range(0, n_out):\r\n","    cols.append(df.shift(-i))\r\n","    if i == 0:\r\n","      names += [('var%d(t)' % (j+1)) for j in range(n_vars)]\r\n","    else:\r\n","      names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]\r\n","  # put it all together\r\n","  agg = pd.concat(cols, axis=1)\r\n","  agg.columns = names\r\n","  # drop rows with NaN values\r\n","  if dropnan:\r\n","    agg.dropna(inplace=True)\r\n","  return pd.DataFrame(agg.astype('float32'))"],"execution_count":27,"outputs":[]},{"cell_type":"code","metadata":{"id":"c46oMGAs_Ile","executionInfo":{"status":"ok","timestamp":1607937726408,"user_tz":360,"elapsed":1106,"user":{"displayName":"Zr Xiang","photoUrl":"","userId":"10347344196072452629"}}},"source":["def train_valid_test_split(data, hours_of_history, hours_to_predict, parameters_included):\r\n","  y_list = list(range(parameters_included*hours_of_history+parameters_included-1, parameters_included*(hours_of_history+hours_to_predict)+parameters_included-1, parameters_included))\r\n","  all_list = list(range(0, parameters_included*(hours_of_history+hours_to_predict)))\r\n","  x_list = [item for item in all_list if item not in y_list]\r\n","\r\n","  data_train = data.iloc[:52608,:] # the first 6 years for training/validation\r\n","  data_test = data.iloc[52608:,:] # the last 1 years for test evaluation\r\n","  data_train.dropna(inplace=True)\r\n","  data_test.dropna(inplace=True)\r\n","\r\n","  x_train_valid = data_train.iloc[:,x_list].values\r\n","  x_test = data_test.iloc[:,x_list].values\r\n","  y_train_valid = data_train.iloc[:,y_list].values\r\n","  y_test = data_test.iloc[:,y_list].values\r\n","\r\n","  x_valid, x_train, y_valid, y_train = train_test_split(x_train_valid, y_train_valid, test_size=0.4, shuffle= False) # the first 60% data in the first 6 years used for training and the rest 40% used for validation.\r\n","\r\n","  return x_train, x_valid, x_test, y_train, y_valid, y_test"],"execution_count":28,"outputs":[]},{"cell_type":"code","metadata":{"id":"kFW20INaHjoO","executionInfo":{"status":"ok","timestamp":1607937726411,"user_tz":360,"elapsed":1087,"user":{"displayName":"Zr Xiang","photoUrl":"","userId":"10347344196072452629"}}},"source":["def prepare_data(station_id, hours_of_history, hours_to_predict, parameters_included):\r\n","  # load the data\r\n","  data = pd.read_csv('./data/'+str(station_id)+'_data.csv').iloc[:,1:]\r\n","\r\n","  # simple min-max scaling. Other pretreatments such as normalization also work.\r\n","  scaler = MinMaxScaler()\r\n","  scaler.fit(data.iloc[:52608,:]) # min-max scaling without the test dataset.\r\n","  q_max = np.max(data.iloc[:52608,2]) # manually check the maximum and minimum discharge\r\n","  q_min = np.min(data.iloc[:52608,2])\r\n","  data_scaled = scaler.transform(data)\r\n","\r\n","  # data split\r\n","  data_sequence = series_to_supervised(data_scaled, hours_of_history, hours_to_predict)\r\n","  x_train, x_valid, x_test, y_train, y_valid, y_test = train_valid_test_split(data_sequence, hours_of_history, hours_to_predict, parameters_included)\r\n","\r\n","  # Split data into 2 parts for encoder (history) and decoder(future).\r\n","  x_train_encoder = x_train[:,:hours_of_history*parameters_included].reshape(-1, hours_of_history, parameters_included)\r\n","  x_train_decoder = x_train[:,hours_of_history*parameters_included:].reshape(-1, hours_to_predict, parameters_included-1)\r\n","  x_valid_encoder = x_valid[:,:hours_of_history*parameters_included].reshape(-1, hours_of_history, parameters_included)\r\n","  x_valid_decoder = x_valid[:,hours_of_history*parameters_included:].reshape(-1, hours_to_predict, parameters_included-1)\r\n","  x_test_encoder = x_test[:,:hours_of_history*parameters_included].reshape(-1, hours_of_history, parameters_included)\r\n","  x_test_decoder = x_test[:,hours_of_history*parameters_included:].reshape(-1, hours_to_predict, parameters_included-1)\r\n","\r\n","  return x_train_encoder, x_train_decoder, x_valid_encoder, x_valid_decoder, x_test_encoder, x_test_decoder, y_train, y_valid, y_test, q_max, q_min"],"execution_count":29,"outputs":[]},{"cell_type":"code","metadata":{"id":"b_MCNzE8a44-","executionInfo":{"status":"ok","timestamp":1607937726412,"user_tz":360,"elapsed":1071,"user":{"displayName":"Zr Xiang","photoUrl":"","userId":"10347344196072452629"}}},"source":["# define custome loss function (you can use the simple 'mse' as well)\r\n","def nseloss(y_true, y_pred):\r\n","  return K.sum((y_pred-y_true)**2)/K.sum((y_true-K.mean(y_true))**2)"],"execution_count":30,"outputs":[]},{"cell_type":"code","metadata":{"id":"Vy6tFQRBRuUD","executionInfo":{"status":"ok","timestamp":1607937726412,"user_tz":360,"elapsed":1056,"user":{"displayName":"Zr Xiang","photoUrl":"","userId":"10347344196072452629"}}},"source":["def NRM(hours_of_history, hours_to_predict, parameters_included):\r\n","  \r\n","  # design network\r\n","  dim_dense = [128, 64, 64, 32, 32]\r\n","  drop = 0.2\r\n","\r\n","  encoder_input = Input(shape=(hours_of_history,parameters_included))\r\n","  encoder_GRU1 = GRU(32, return_state=True, return_sequences=True)\r\n","  encoder_output1, encoder_hc1 = encoder_GRU1(encoder_input)\r\n","  encoder_GRU2 = GRU(32, return_state=True, return_sequences=True)\r\n","  encoder_output2, encoder_hc2 = encoder_GRU2(encoder_output1)\r\n","  encoder_GRU3 = GRU(32, return_state=True, return_sequences=True)\r\n","  encoder_output3, encoder_hc3 = encoder_GRU3(encoder_output2)\r\n","  encoder_GRU4 = GRU(32, return_state=True, return_sequences=True)\r\n","  encoder_output4, encoder_hc4 = encoder_GRU4(encoder_output3)\r\n","  encoder_GRU5 = GRU(32, return_state=True)\r\n","  encoder_output5, encoder_hc5 = encoder_GRU5(encoder_output4)\r\n","\r\n","  decoder_input = Input(shape=(hours_to_predict,parameters_included-1))\r\n","  decoder_GRU1 = GRU(32, return_sequences=True)\r\n","  decoder_GRU2 = GRU(32, return_sequences=True)\r\n","  decoder_GRU3 = GRU(32, return_sequences=True)\r\n","  decoder_GRU4 = GRU(32, return_sequences=True)\r\n","  decoder_GRU5 = GRU(32, return_sequences=True)\r\n","  x = decoder_GRU1(decoder_input, initial_state=encoder_hc1)\r\n","  x = decoder_GRU2(x, initial_state=encoder_hc2)\r\n","  x = decoder_GRU3(x, initial_state=encoder_hc3)\r\n","  x = decoder_GRU4(x, initial_state=encoder_hc4)\r\n","  x = decoder_GRU5(x, initial_state=encoder_hc5)\r\n","\r\n","  for dim in dim_dense:\r\n","    x = TimeDistributed(Dense(dim, activation='tanh'))(x) # or relu\r\n","    x = TimeDistributed(Dropout(drop))(x)\r\n","  main_out = TimeDistributed(Dense(1, activation='linear'))(x) # relu as the last activation works (only) for the min-max scalling so there is no negative output\r\n","  main_out = Flatten()(main_out)\r\n","  model = Model(inputs=[encoder_input, decoder_input], outputs=main_out)\r\n","  return model"],"execution_count":31,"outputs":[]},{"cell_type":"code","metadata":{"id":"PvTU6Ob4cc8A","executionInfo":{"status":"ok","timestamp":1607937726413,"user_tz":360,"elapsed":1039,"user":{"displayName":"Zr Xiang","photoUrl":"","userId":"10347344196072452629"}}},"source":["# identify KGE, NSE for evaluation\r\n","def nse(y_true, y_pred):\r\n","  return 1-np.sum((y_pred-y_true)**2)/np.sum((y_true-np.mean(y_true))**2)\r\n","  \r\n","def kge(y_true, y_pred):\r\n","  kge_r = np.corrcoef(y_true,y_pred)[1][0]\r\n","  kge_a = np.std(y_pred)/np.std(y_true)\r\n","  kge_b = np.mean(y_pred)/np.mean(y_true)\r\n","  return 1-np.sqrt((kge_r-1)**2+(kge_a-1)**2+(kge_b-1)**2)\r\n"],"execution_count":32,"outputs":[]},{"cell_type":"code","metadata":{"id":"MptO3dI7Hucj","executionInfo":{"status":"ok","timestamp":1607937726414,"user_tz":360,"elapsed":1021,"user":{"displayName":"Zr Xiang","photoUrl":"","userId":"10347344196072452629"}}},"source":["def main():\r\n","  # parameters\r\n","  station_id = 521\r\n","  hours_to_predict = 120\r\n","  hours_of_history = 72\r\n","  parameters_included = 3\r\n","\r\n","  batch_size = 64\r\n","  lr = 0.0001\r\n","  epochs = 300\r\n","  test_name = './'+str(station_id)+'_model2_'\r\n","\r\n","  # load data\r\n","  x_train_encoder, x_train_decoder, x_valid_encoder, x_valid_decoder, x_test_encoder, x_test_decoder, y_train, y_valid, y_test, q_max, q_min = prepare_data(station_id, hours_of_history, hours_to_predict, parameters_included)\r\n","  model2 = NRM(hours_of_history, hours_to_predict, parameters_included)\r\n","\r\n","  # compile settings\r\n","  reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=10, cooldown=200, min_lr=1e-8)\r\n","  earlystoping = EarlyStopping(monitor='val_loss', min_delta=0, patience=20, verbose=1, mode='auto')\r\n","  checkpoint = ModelCheckpoint(test_name+'model.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min')\r\n","  optimizer = RMSprop(lr=lr)\r\n","  model2.compile(optimizer=optimizer, loss=nseloss) # in paper, we used the customized \"nseloss\" as model 1 did. however, here we show mse as an example.\r\n","\r\n","  # train model\r\n","  history = model2.fit([x_train_encoder, x_train_decoder], y_train, epochs=epochs, batch_size=batch_size,\r\n","              validation_data=([x_valid_encoder, x_valid_decoder], y_valid), callbacks=[reduce_lr, earlystoping, checkpoint], verbose=1)\r\n","\r\n","  # save training loss\r\n","  loss_train = history.history['loss']\r\n","  loss_valid = history.history['val_loss']\r\n","  loss_train = pd.DataFrame({'TrainLoss':loss_train})\r\n","  loss_valid = pd.DataFrame({'TestLoss':loss_valid})\r\n","  LossEpoches = pd.concat([loss_train, loss_valid], axis=1)\r\n","  LossEpoches.to_csv(test_name+'loss.csv', index = True)\r\n","\r\n","  # Final Test Review\r\n","  model2.load_weights(test_name+'model.h5')\r\n","\r\n","  y_model_scaled = model2.predict([x_test_encoder,x_test_decoder])\r\n","  y_model = y_model_scaled*(q_max-q_min)+q_min\r\n","  y_test = y_test*(q_max-q_min)+q_min\r\n","\r\n","  # hourly evaluation\r\n","  NSEs=[]\r\n","  KGEs=[]\r\n","  for x in range(0, 120):\r\n","    y_pred = y_model[:,x]\r\n","    y_True = y_test[:,x]\r\n","    NSEs.append(nse(y_True,y_pred))\r\n","    KGEs.append(kge(y_True,y_pred))\r\n","    \r\n","  NSEs=pd.DataFrame(NSEs)\r\n","  NSEs.columns = ['NSE_Test']\r\n","  KGEs=pd.DataFrame(KGEs)\r\n","  KGEs.columns = ['KGE_Test']\r\n","    \r\n","  eva = pd.concat([NSEs, KGEs], axis=1)\r\n","  eva.to_csv(test_name+'eva.csv', index = True)\r\n"," "],"execution_count":33,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"o03f8xK7ao9g","executionInfo":{"status":"ok","timestamp":1607939020541,"user_tz":360,"elapsed":1295108,"user":{"displayName":"Zr Xiang","photoUrl":"","userId":"10347344196072452629"}},"outputId":"62e69b2c-28f6-4a98-cbf6-aeb1d065a61a"},"source":["if __name__ == \"__main__\":\r\n","  main()\r\n"],"execution_count":34,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  \n","/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:9: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  if __name__ == '__main__':\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch 1/300\n","273/274 [============================>.] - ETA: 0s - loss: 1.0383\n","Epoch 00001: val_loss improved from inf to 5531.61035, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 20s 73ms/step - loss: 1.0383 - val_loss: 5531.6104\n","Epoch 2/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.8461\n","Epoch 00002: val_loss improved from 5531.61035 to 3695.69385, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 18s 66ms/step - loss: 0.8463 - val_loss: 3695.6938\n","Epoch 3/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.7912\n","Epoch 00003: val_loss did not improve from 3695.69385\n","274/274 [==============================] - 18s 65ms/step - loss: 0.7910 - val_loss: 5432.3809\n","Epoch 4/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.7527\n","Epoch 00004: val_loss did not improve from 3695.69385\n","274/274 [==============================] - 18s 65ms/step - loss: 0.7529 - val_loss: 6673.6909\n","Epoch 5/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.7128\n","Epoch 00005: val_loss did not improve from 3695.69385\n","274/274 [==============================] - 18s 64ms/step - loss: 0.7137 - val_loss: 12068.3467\n","Epoch 6/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.6887\n","Epoch 00006: val_loss improved from 3695.69385 to 3192.24487, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 18s 66ms/step - loss: 0.6889 - val_loss: 3192.2449\n","Epoch 7/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.6627\n","Epoch 00007: val_loss did not improve from 3192.24487\n","274/274 [==============================] - 18s 64ms/step - loss: 0.6626 - val_loss: 7154.7251\n","Epoch 8/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.6024\n","Epoch 00008: val_loss improved from 3192.24487 to 562.10370, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 18s 65ms/step - loss: 0.6037 - val_loss: 562.1037\n","Epoch 9/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.5706\n","Epoch 00009: val_loss did not improve from 562.10370\n","274/274 [==============================] - 17s 64ms/step - loss: 0.5706 - val_loss: 713.0746\n","Epoch 10/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.5495\n","Epoch 00010: val_loss improved from 562.10370 to 435.07776, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 18s 66ms/step - loss: 0.5502 - val_loss: 435.0778\n","Epoch 11/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.5143\n","Epoch 00011: val_loss did not improve from 435.07776\n","274/274 [==============================] - 18s 64ms/step - loss: 0.5147 - val_loss: 6996.6069\n","Epoch 12/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.5065\n","Epoch 00012: val_loss did not improve from 435.07776\n","274/274 [==============================] - 17s 64ms/step - loss: 0.5068 - val_loss: 11011.1992\n","Epoch 13/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.4794\n","Epoch 00013: val_loss did not improve from 435.07776\n","274/274 [==============================] - 18s 64ms/step - loss: 0.4792 - val_loss: 6247.5283\n","Epoch 14/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.4617\n","Epoch 00014: val_loss did not improve from 435.07776\n","274/274 [==============================] - 18s 64ms/step - loss: 0.4618 - val_loss: 3916.8425\n","Epoch 15/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.4354\n","Epoch 00015: val_loss did not improve from 435.07776\n","274/274 [==============================] - 18s 64ms/step - loss: 0.4355 - val_loss: 18663.1016\n","Epoch 16/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.4126\n","Epoch 00016: val_loss did not improve from 435.07776\n","274/274 [==============================] - 18s 64ms/step - loss: 0.4130 - val_loss: 1051.7140\n","Epoch 17/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.3938\n","Epoch 00017: val_loss did not improve from 435.07776\n","274/274 [==============================] - 17s 64ms/step - loss: 0.3936 - val_loss: 3078.7734\n","Epoch 18/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.3684\n","Epoch 00018: val_loss did not improve from 435.07776\n","274/274 [==============================] - 18s 65ms/step - loss: 0.3686 - val_loss: 1717.1943\n","Epoch 19/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.3399\n","Epoch 00019: val_loss did not improve from 435.07776\n","274/274 [==============================] - 18s 64ms/step - loss: 0.3397 - val_loss: 2786.4436\n","Epoch 20/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.3183\n","Epoch 00020: val_loss did not improve from 435.07776\n","274/274 [==============================] - 18s 64ms/step - loss: 0.3182 - val_loss: 1953.2253\n","Epoch 21/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.2393\n","Epoch 00021: val_loss did not improve from 435.07776\n","274/274 [==============================] - 18s 64ms/step - loss: 0.2392 - val_loss: 915.4641\n","Epoch 22/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.2233\n","Epoch 00022: val_loss did not improve from 435.07776\n","274/274 [==============================] - 18s 64ms/step - loss: 0.2235 - val_loss: 1178.0759\n","Epoch 23/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.2162\n","Epoch 00023: val_loss did not improve from 435.07776\n","274/274 [==============================] - 17s 64ms/step - loss: 0.2162 - val_loss: 478.8888\n","Epoch 24/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.2070\n","Epoch 00024: val_loss improved from 435.07776 to 347.00613, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 18s 65ms/step - loss: 0.2069 - val_loss: 347.0061\n","Epoch 25/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.2009\n","Epoch 00025: val_loss improved from 347.00613 to 318.66760, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 18s 65ms/step - loss: 0.2009 - val_loss: 318.6676\n","Epoch 26/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.2035\n","Epoch 00026: val_loss improved from 318.66760 to 176.74379, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 18s 66ms/step - loss: 0.2035 - val_loss: 176.7438\n","Epoch 27/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1990\n","Epoch 00027: val_loss did not improve from 176.74379\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1989 - val_loss: 190.3753\n","Epoch 28/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1989\n","Epoch 00028: val_loss improved from 176.74379 to 160.90015, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 18s 65ms/step - loss: 0.1989 - val_loss: 160.9001\n","Epoch 29/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1926\n","Epoch 00029: val_loss did not improve from 160.90015\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1927 - val_loss: 784.7363\n","Epoch 30/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1901\n","Epoch 00030: val_loss did not improve from 160.90015\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1905 - val_loss: 210.1701\n","Epoch 31/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1920\n","Epoch 00031: val_loss did not improve from 160.90015\n","274/274 [==============================] - 17s 63ms/step - loss: 0.1920 - val_loss: 333.5205\n","Epoch 32/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1869\n","Epoch 00032: val_loss did not improve from 160.90015\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1869 - val_loss: 269.3921\n","Epoch 33/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1864\n","Epoch 00033: val_loss did not improve from 160.90015\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1865 - val_loss: 448.0331\n","Epoch 34/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1878\n","Epoch 00034: val_loss did not improve from 160.90015\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1878 - val_loss: 387.1707\n","Epoch 35/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1856\n","Epoch 00035: val_loss did not improve from 160.90015\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1855 - val_loss: 508.9729\n","Epoch 36/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1836\n","Epoch 00036: val_loss did not improve from 160.90015\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1837 - val_loss: 1087.8058\n","Epoch 37/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1837\n","Epoch 00037: val_loss improved from 160.90015 to 139.49078, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 18s 65ms/step - loss: 0.1837 - val_loss: 139.4908\n","Epoch 38/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1860\n","Epoch 00038: val_loss improved from 139.49078 to 134.35907, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 18s 66ms/step - loss: 0.1860 - val_loss: 134.3591\n","Epoch 39/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1821\n","Epoch 00039: val_loss improved from 134.35907 to 132.49643, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 18s 66ms/step - loss: 0.1820 - val_loss: 132.4964\n","Epoch 40/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1801\n","Epoch 00040: val_loss did not improve from 132.49643\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1801 - val_loss: 184.7363\n","Epoch 41/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1821\n","Epoch 00041: val_loss did not improve from 132.49643\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1823 - val_loss: 738.2755\n","Epoch 42/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1798\n","Epoch 00042: val_loss did not improve from 132.49643\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1798 - val_loss: 231.5997\n","Epoch 43/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1800\n","Epoch 00043: val_loss improved from 132.49643 to 117.69077, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 18s 65ms/step - loss: 0.1803 - val_loss: 117.6908\n","Epoch 44/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1811\n","Epoch 00044: val_loss did not improve from 117.69077\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1811 - val_loss: 169.7006\n","Epoch 45/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1777\n","Epoch 00045: val_loss did not improve from 117.69077\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1776 - val_loss: 280.2709\n","Epoch 46/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1761\n","Epoch 00046: val_loss did not improve from 117.69077\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1760 - val_loss: 128.9448\n","Epoch 47/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1738\n","Epoch 00047: val_loss did not improve from 117.69077\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1738 - val_loss: 162.6325\n","Epoch 48/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1731\n","Epoch 00048: val_loss did not improve from 117.69077\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1731 - val_loss: 1047.7728\n","Epoch 49/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1738\n","Epoch 00049: val_loss did not improve from 117.69077\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1739 - val_loss: 319.1815\n","Epoch 50/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1781\n","Epoch 00050: val_loss did not improve from 117.69077\n","274/274 [==============================] - 17s 63ms/step - loss: 0.1782 - val_loss: 127.1995\n","Epoch 51/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1734\n","Epoch 00051: val_loss did not improve from 117.69077\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1736 - val_loss: 470.1798\n","Epoch 52/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1761\n","Epoch 00052: val_loss improved from 117.69077 to 113.48692, saving model to ./521_model2_model.h5\n","274/274 [==============================] - 18s 66ms/step - loss: 0.1761 - val_loss: 113.4869\n","Epoch 53/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1726\n","Epoch 00053: val_loss did not improve from 113.48692\n","274/274 [==============================] - 18s 65ms/step - loss: 0.1725 - val_loss: 132.5643\n","Epoch 54/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1692\n","Epoch 00054: val_loss did not improve from 113.48692\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1694 - val_loss: 261.3999\n","Epoch 55/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1701\n","Epoch 00055: val_loss did not improve from 113.48692\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1702 - val_loss: 246.3953\n","Epoch 56/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1713\n","Epoch 00056: val_loss did not improve from 113.48692\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1712 - val_loss: 138.2197\n","Epoch 57/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1666\n","Epoch 00057: val_loss did not improve from 113.48692\n","274/274 [==============================] - 17s 63ms/step - loss: 0.1665 - val_loss: 165.8296\n","Epoch 58/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1705\n","Epoch 00058: val_loss did not improve from 113.48692\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1707 - val_loss: 458.5744\n","Epoch 59/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1698\n","Epoch 00059: val_loss did not improve from 113.48692\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1698 - val_loss: 201.9707\n","Epoch 60/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1661\n","Epoch 00060: val_loss did not improve from 113.48692\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1663 - val_loss: 142.6167\n","Epoch 61/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1681\n","Epoch 00061: val_loss did not improve from 113.48692\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1682 - val_loss: 259.5958\n","Epoch 62/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1621\n","Epoch 00062: val_loss did not improve from 113.48692\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1621 - val_loss: 224.2349\n","Epoch 63/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1658\n","Epoch 00063: val_loss did not improve from 113.48692\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1657 - val_loss: 137.7745\n","Epoch 64/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1624\n","Epoch 00064: val_loss did not improve from 113.48692\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1624 - val_loss: 186.2458\n","Epoch 65/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1672\n","Epoch 00065: val_loss did not improve from 113.48692\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1672 - val_loss: 135.4294\n","Epoch 66/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1623\n","Epoch 00066: val_loss did not improve from 113.48692\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1623 - val_loss: 205.2856\n","Epoch 67/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1619\n","Epoch 00067: val_loss did not improve from 113.48692\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1619 - val_loss: 245.3801\n","Epoch 68/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1617\n","Epoch 00068: val_loss did not improve from 113.48692\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1617 - val_loss: 247.6842\n","Epoch 69/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1625\n","Epoch 00069: val_loss did not improve from 113.48692\n","274/274 [==============================] - 17s 64ms/step - loss: 0.1626 - val_loss: 808.0713\n","Epoch 70/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1617\n","Epoch 00070: val_loss did not improve from 113.48692\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1619 - val_loss: 735.3866\n","Epoch 71/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1607\n","Epoch 00071: val_loss did not improve from 113.48692\n","274/274 [==============================] - 18s 65ms/step - loss: 0.1606 - val_loss: 147.9675\n","Epoch 72/300\n","273/274 [============================>.] - ETA: 0s - loss: 0.1615\n","Epoch 00072: val_loss did not improve from 113.48692\n","274/274 [==============================] - 18s 64ms/step - loss: 0.1616 - val_loss: 136.9624\n","Epoch 00072: early stopping\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"s2fcQj_ET3oG","executionInfo":{"status":"ok","timestamp":1607939020551,"user_tz":360,"elapsed":1295091,"user":{"displayName":"Zr Xiang","photoUrl":"","userId":"10347344196072452629"}}},"source":[""],"execution_count":34,"outputs":[]}]}