[1]:
%matplotlib inline
D:\C\Anaconda3\envs\tfcpu27_py39\lib\site-packages\numpy\_distributor_init.py:30: UserWarning: loaded more than 1 DLL from .libs:
D:\C\Anaconda3\envs\tfcpu27_py39\lib\site-packages\numpy\.libs\libopenblas.EL2C6PLE4ZYW3ECEVIV3OXXGRN2NRFM2.gfortran-win_amd64.dll
D:\C\Anaconda3\envs\tfcpu27_py39\lib\site-packages\numpy\.libs\libopenblas.GK7GX5KEQ4F6UYO3P26ULGBQYHGQO7J4.gfortran-win_amd64.dll
warnings.warn("loaded more than 1 DLL from .libs:"
Temporal Fusion Transformer
This notebook shows usage of temporal fusion transformer model. The TFT model was developed by a team from Google led by Lim et al. This neural network architecture combines the power of LSTM for extract temporal features from time series data and transformer architecture to further compliment LSTM and make the architecture interpretable. Although the TFT model can take both categorical and numerical features but in this notebook we will consider only numerical features.
[1]:
# Some features used in this notebook are not available in the latest release of ai4water from pip which is 1.06
# at the moment. They will be available in ai4water's next release in 1.07. Since 1.07 is not currently available on
# pip, we will install ai4water from github using the following command.
# try:
# import ai4water
# except (ImportError, ModuleNotFoundError):
# !pip install git+https://github.com/AtrCheema/AI4Water.git@50ec3cf6ec281de104ca85f9f748dd0367235f62
[41]:
import tensorflow as tf
tf.compat.v1.disable_eager_execution()
import numpy as np
from easy_mpl import imshow
from ai4water import Model
from ai4water.datasets import MtropicsLaos
from ai4water.utils.utils import get_version_info
[2]:
for k,v in get_version_info().items():
print(f"{k} version: {v}")
python version: 3.7.7 (default, Apr 15 2020, 05:09:04) [MSC v.1916 64 bit (AMD64)]
os version: nt
ai4water version: 1.07
lightgbm version: 3.2.1
catboost version: 0.26
xgboost version: 1.2.1
easy_mpl version: 0.21.3
SeqMetrics version: 1.3.4
tensorflow version: 1.15.0
tensorflow.python.keras.api._v1.keras version: 2.2.4-tf
torch version: 1.8.1+cpu
numpy version: 1.18.1
pandas version: 1.3.5
matplotlib version: 3.1.3
h5py version: 2.10.0
sklearn version: 0.22.2.post1
shapefile version: 2.1.3
fiona version: 1.8.11
xarray version: 0.19.0
netCDF4 version: 1.5.6
optuna version: 2.7.0
skopt version: 0.7.4
plotly version: 4.14.1
seaborn version: 0.10.0
[3]:
from ai4water import Model
from ai4water.models import TFT
from ai4water.datasets import CAMELS_AUS
[4]:
dataset = CAMELS_AUS(path="F:\\data\\CAMELS\\CAMELS_AUS")
[5]:
inputs = ['et_morton_point_SILO',
'precipitation_AWAP',
'tmax_AWAP',
'tmin_AWAP',
'vprp_AWAP',
'rh_tmax_SILO',
'rh_tmin_SILO'
]
outputs = ['streamflow_MLd']
data = dataset.fetch('401203', dynamic_features=inputs+outputs, as_dataframe=True)
data = data.unstack()
data.columns = [a[1] for a in data.columns.to_flat_index()]
data.shape
[5]:
(21184, 8)
[6]:
skew_inputs = [
'precipitation_AWAP',
'rh_tmin_SILO'
]
normal_inputs = ['et_morton_point_SILO',
'tmax_AWAP',
'tmin_AWAP',
'vprp_AWAP',
'rh_tmax_SILO',
]
[8]:
model = Model(
model = TFT(
input_shape=(15, len(inputs)),
hidden_units=80,
num_heads=6
),
input_features=inputs,
output_features=outputs,
epochs=700,
ts_args={'lookback': 15},
lr = 1.8928566321671455e-05,
batch_size = 16,
x_transformation=[{'method': 'robust', 'features': normal_inputs},
{'method': 'log', "replace_zeros": True, 'features': skew_inputs}],
y_transformation={'method': 'robust', 'features': outputs},
)
building DL model for
regression problem using Model
Model: "model_1"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
Input (InputLayer) [(None, 15, 7)] 0
__________________________________________________________________________________________________
tf_op_layer_strided_slice (Tens [(None, 15, 7)] 0 Input[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_1 (Te [(None, 15, 1)] 0 tf_op_layer_strided_slice[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_2 (Te [(None, 15, 1)] 0 tf_op_layer_strided_slice[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_3 (Te [(None, 15, 1)] 0 tf_op_layer_strided_slice[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_4 (Te [(None, 15, 1)] 0 tf_op_layer_strided_slice[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_5 (Te [(None, 15, 1)] 0 tf_op_layer_strided_slice[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_6 (Te [(None, 15, 1)] 0 tf_op_layer_strided_slice[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_7 (Te [(None, 15, 1)] 0 tf_op_layer_strided_slice[0][0]
__________________________________________________________________________________________________
time_distributed (TimeDistribut (None, 15, 80) 160 tf_op_layer_strided_slice_1[0][0]
__________________________________________________________________________________________________
time_distributed_1 (TimeDistrib (None, 15, 80) 160 tf_op_layer_strided_slice_2[0][0]
__________________________________________________________________________________________________
time_distributed_2 (TimeDistrib (None, 15, 80) 160 tf_op_layer_strided_slice_3[0][0]
__________________________________________________________________________________________________
time_distributed_3 (TimeDistrib (None, 15, 80) 160 tf_op_layer_strided_slice_4[0][0]
__________________________________________________________________________________________________
time_distributed_4 (TimeDistrib (None, 15, 80) 160 tf_op_layer_strided_slice_5[0][0]
__________________________________________________________________________________________________
time_distributed_5 (TimeDistrib (None, 15, 80) 160 tf_op_layer_strided_slice_6[0][0]
__________________________________________________________________________________________________
time_distributed_6 (TimeDistrib (None, 15, 80) 160 tf_op_layer_strided_slice_7[0][0]
__________________________________________________________________________________________________
tf_op_layer_stack (TensorFlowOp [(None, 15, 80, 7)] 0 time_distributed[0][0]
time_distributed_1[0][0]
time_distributed_2[0][0]
time_distributed_3[0][0]
time_distributed_4[0][0]
time_distributed_5[0][0]
time_distributed_6[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_8 (Te [(None, 15, 80, 7)] 0 tf_op_layer_stack[0][0]
__________________________________________________________________________________________________
tf_op_layer_Reshape (TensorFlow [(None, 15, 560)] 0 tf_op_layer_strided_slice_8[0][0]
__________________________________________________________________________________________________
ff_GRN_with_history (Dense) (None, 15, 80) 44880 tf_op_layer_Reshape[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_9 (Te [(None, 15, 80)] 0 tf_op_layer_strided_slice_8[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_10 (T [(None, 15, 80)] 0 tf_op_layer_strided_slice_8[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_11 (T [(None, 15, 80)] 0 tf_op_layer_strided_slice_8[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_12 (T [(None, 15, 80)] 0 tf_op_layer_strided_slice_8[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_13 (T [(None, 15, 80)] 0 tf_op_layer_strided_slice_8[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_14 (T [(None, 15, 80)] 0 tf_op_layer_strided_slice_8[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_15 (T [(None, 15, 80)] 0 tf_op_layer_strided_slice_8[0][0]
__________________________________________________________________________________________________
GRN_with_history_elu (Activatio (None, 15, 80) 0 ff_GRN_with_history[0][0]
__________________________________________________________________________________________________
ff_GRN_with_history_for_0 (Dens (None, 15, 80) 6480 tf_op_layer_strided_slice_9[0][0]
__________________________________________________________________________________________________
ff_GRN_with_history_for_1 (Dens (None, 15, 80) 6480 tf_op_layer_strided_slice_10[0][0
__________________________________________________________________________________________________
ff_GRN_with_history_for_2 (Dens (None, 15, 80) 6480 tf_op_layer_strided_slice_11[0][0
__________________________________________________________________________________________________
ff_GRN_with_history_for_3 (Dens (None, 15, 80) 6480 tf_op_layer_strided_slice_12[0][0
__________________________________________________________________________________________________
ff_GRN_with_history_for_4 (Dens (None, 15, 80) 6480 tf_op_layer_strided_slice_13[0][0
__________________________________________________________________________________________________
ff_GRN_with_history_for_5 (Dens (None, 15, 80) 6480 tf_op_layer_strided_slice_14[0][0
__________________________________________________________________________________________________
ff_GRN_with_history_for_6 (Dens (None, 15, 80) 6480 tf_op_layer_strided_slice_15[0][0
__________________________________________________________________________________________________
GRN_with_history_LastDense (Den (None, 15, 80) 6480 GRN_with_history_elu[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_0_elu (Act (None, 15, 80) 0 ff_GRN_with_history_for_0[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_1_elu (Act (None, 15, 80) 0 ff_GRN_with_history_for_1[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_2_elu (Act (None, 15, 80) 0 ff_GRN_with_history_for_2[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_3_elu (Act (None, 15, 80) 0 ff_GRN_with_history_for_3[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_4_elu (Act (None, 15, 80) 0 ff_GRN_with_history_for_4[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_5_elu (Act (None, 15, 80) 0 ff_GRN_with_history_for_5[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_6_elu (Act (None, 15, 80) 0 ff_GRN_with_history_for_6[0][0]
__________________________________________________________________________________________________
Dropout_GRN_with_history (Dropo (None, 15, 80) 0 GRN_with_history_LastDense[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_0_LastDens (None, 15, 80) 6480 GRN_with_history_for_0_elu[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_1_LastDens (None, 15, 80) 6480 GRN_with_history_for_1_elu[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_2_LastDens (None, 15, 80) 6480 GRN_with_history_for_2_elu[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_3_LastDens (None, 15, 80) 6480 GRN_with_history_for_3_elu[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_4_LastDens (None, 15, 80) 6480 GRN_with_history_for_4_elu[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_5_LastDens (None, 15, 80) 6480 GRN_with_history_for_5_elu[0][0]
__________________________________________________________________________________________________
GRN_with_history_for_6_LastDens (None, 15, 80) 6480 GRN_with_history_for_6_elu[0][0]
__________________________________________________________________________________________________
gating_act_GRN_with_history (De (None, 15, 7) 567 Dropout_GRN_with_history[0][0]
__________________________________________________________________________________________________
gating_GRN_with_history (Dense) (None, 15, 7) 567 Dropout_GRN_with_history[0][0]
__________________________________________________________________________________________________
Dropout_GRN_with_history_for_0 (None, 15, 80) 0 GRN_with_history_for_0_LastDense[
__________________________________________________________________________________________________
Dropout_GRN_with_history_for_1 (None, 15, 80) 0 GRN_with_history_for_1_LastDense[
__________________________________________________________________________________________________
Dropout_GRN_with_history_for_2 (None, 15, 80) 0 GRN_with_history_for_2_LastDense[
__________________________________________________________________________________________________
Dropout_GRN_with_history_for_3 (None, 15, 80) 0 GRN_with_history_for_3_LastDense[
__________________________________________________________________________________________________
Dropout_GRN_with_history_for_4 (None, 15, 80) 0 GRN_with_history_for_4_LastDense[
__________________________________________________________________________________________________
Dropout_GRN_with_history_for_5 (None, 15, 80) 0 GRN_with_history_for_5_LastDense[
__________________________________________________________________________________________________
Dropout_GRN_with_history_for_6 (None, 15, 80) 0 GRN_with_history_for_6_LastDense[
__________________________________________________________________________________________________
skip_connection_GRN_with_histor (None, 15, 7) 3927 tf_op_layer_Reshape[0][0]
__________________________________________________________________________________________________
MulGating_GRN_with_history (Mul (None, 15, 7) 0 gating_act_GRN_with_history[0][0]
gating_GRN_with_history[0][0]
__________________________________________________________________________________________________
gating_act_GRN_with_history_for (None, 15, 80) 6480 Dropout_GRN_with_history_for_0[0]
__________________________________________________________________________________________________
gating_GRN_with_history_for_0 ( (None, 15, 80) 6480 Dropout_GRN_with_history_for_0[0]
__________________________________________________________________________________________________
gating_act_GRN_with_history_for (None, 15, 80) 6480 Dropout_GRN_with_history_for_1[0]
__________________________________________________________________________________________________
gating_GRN_with_history_for_1 ( (None, 15, 80) 6480 Dropout_GRN_with_history_for_1[0]
__________________________________________________________________________________________________
gating_act_GRN_with_history_for (None, 15, 80) 6480 Dropout_GRN_with_history_for_2[0]
__________________________________________________________________________________________________
gating_GRN_with_history_for_2 ( (None, 15, 80) 6480 Dropout_GRN_with_history_for_2[0]
__________________________________________________________________________________________________
gating_act_GRN_with_history_for (None, 15, 80) 6480 Dropout_GRN_with_history_for_3[0]
__________________________________________________________________________________________________
gating_GRN_with_history_for_3 ( (None, 15, 80) 6480 Dropout_GRN_with_history_for_3[0]
__________________________________________________________________________________________________
gating_act_GRN_with_history_for (None, 15, 80) 6480 Dropout_GRN_with_history_for_4[0]
__________________________________________________________________________________________________
gating_GRN_with_history_for_4 ( (None, 15, 80) 6480 Dropout_GRN_with_history_for_4[0]
__________________________________________________________________________________________________
gating_act_GRN_with_history_for (None, 15, 80) 6480 Dropout_GRN_with_history_for_5[0]
__________________________________________________________________________________________________
gating_GRN_with_history_for_5 ( (None, 15, 80) 6480 Dropout_GRN_with_history_for_5[0]
__________________________________________________________________________________________________
gating_act_GRN_with_history_for (None, 15, 80) 6480 Dropout_GRN_with_history_for_6[0]
__________________________________________________________________________________________________
gating_GRN_with_history_for_6 ( (None, 15, 80) 6480 Dropout_GRN_with_history_for_6[0]
__________________________________________________________________________________________________
add_GRN_with_history (Add) (None, 15, 7) 0 skip_connection_GRN_with_history[
MulGating_GRN_with_history[0][0]
__________________________________________________________________________________________________
MulGating_GRN_with_history_for_ (None, 15, 80) 0 gating_act_GRN_with_history_for_0
gating_GRN_with_history_for_0[0][
__________________________________________________________________________________________________
MulGating_GRN_with_history_for_ (None, 15, 80) 0 gating_act_GRN_with_history_for_1
gating_GRN_with_history_for_1[0][
__________________________________________________________________________________________________
MulGating_GRN_with_history_for_ (None, 15, 80) 0 gating_act_GRN_with_history_for_2
gating_GRN_with_history_for_2[0][
__________________________________________________________________________________________________
MulGating_GRN_with_history_for_ (None, 15, 80) 0 gating_act_GRN_with_history_for_3
gating_GRN_with_history_for_3[0][
__________________________________________________________________________________________________
MulGating_GRN_with_history_for_ (None, 15, 80) 0 gating_act_GRN_with_history_for_4
gating_GRN_with_history_for_4[0][
__________________________________________________________________________________________________
MulGating_GRN_with_history_for_ (None, 15, 80) 0 gating_act_GRN_with_history_for_5
gating_GRN_with_history_for_5[0][
__________________________________________________________________________________________________
MulGating_GRN_with_history_for_ (None, 15, 80) 0 gating_act_GRN_with_history_for_6
gating_GRN_with_history_for_6[0][
__________________________________________________________________________________________________
norm_GRN_with_history (LayerNor (None, 15, 7) 14 add_GRN_with_history[0][0]
__________________________________________________________________________________________________
add_GRN_with_history_for_0 (Add (None, 15, 80) 0 tf_op_layer_strided_slice_9[0][0]
MulGating_GRN_with_history_for_0[
__________________________________________________________________________________________________
add_GRN_with_history_for_1 (Add (None, 15, 80) 0 tf_op_layer_strided_slice_10[0][0
MulGating_GRN_with_history_for_1[
__________________________________________________________________________________________________
add_GRN_with_history_for_2 (Add (None, 15, 80) 0 tf_op_layer_strided_slice_11[0][0
MulGating_GRN_with_history_for_2[
__________________________________________________________________________________________________
add_GRN_with_history_for_3 (Add (None, 15, 80) 0 tf_op_layer_strided_slice_12[0][0
MulGating_GRN_with_history_for_3[
__________________________________________________________________________________________________
add_GRN_with_history_for_4 (Add (None, 15, 80) 0 tf_op_layer_strided_slice_13[0][0
MulGating_GRN_with_history_for_4[
__________________________________________________________________________________________________
add_GRN_with_history_for_5 (Add (None, 15, 80) 0 tf_op_layer_strided_slice_14[0][0
MulGating_GRN_with_history_for_5[
__________________________________________________________________________________________________
add_GRN_with_history_for_6 (Add (None, 15, 80) 0 tf_op_layer_strided_slice_15[0][0
MulGating_GRN_with_history_for_6[
__________________________________________________________________________________________________
sparse_history_weights_softmax (None, 15, 7) 0 norm_GRN_with_history[0][0]
__________________________________________________________________________________________________
norm_GRN_with_history_for_0 (La (None, 15, 80) 160 add_GRN_with_history_for_0[0][0]
__________________________________________________________________________________________________
norm_GRN_with_history_for_1 (La (None, 15, 80) 160 add_GRN_with_history_for_1[0][0]
__________________________________________________________________________________________________
norm_GRN_with_history_for_2 (La (None, 15, 80) 160 add_GRN_with_history_for_2[0][0]
__________________________________________________________________________________________________
norm_GRN_with_history_for_3 (La (None, 15, 80) 160 add_GRN_with_history_for_3[0][0]
__________________________________________________________________________________________________
norm_GRN_with_history_for_4 (La (None, 15, 80) 160 add_GRN_with_history_for_4[0][0]
__________________________________________________________________________________________________
norm_GRN_with_history_for_5 (La (None, 15, 80) 160 add_GRN_with_history_for_5[0][0]
__________________________________________________________________________________________________
norm_GRN_with_history_for_6 (La (None, 15, 80) 160 add_GRN_with_history_for_6[0][0]
__________________________________________________________________________________________________
tf_op_layer_ExpandDims (TensorF [(None, 15, 1, 7)] 0 sparse_history_weights_softmax[0]
__________________________________________________________________________________________________
tf_op_layer_stack_1 (TensorFlow [(None, 15, 80, 7)] 0 norm_GRN_with_history_for_0[0][0]
norm_GRN_with_history_for_1[0][0]
norm_GRN_with_history_for_2[0][0]
norm_GRN_with_history_for_3[0][0]
norm_GRN_with_history_for_4[0][0]
norm_GRN_with_history_for_5[0][0]
norm_GRN_with_history_for_6[0][0]
__________________________________________________________________________________________________
sparse_and_transform_history (M (None, 15, 80, 7) 0 tf_op_layer_ExpandDims[0][0]
tf_op_layer_stack_1[0][0]
__________________________________________________________________________________________________
tf_op_layer_Sum (TensorFlowOpLa [(None, 15, 80)] 0 sparse_and_transform_history[0][0
__________________________________________________________________________________________________
history (LSTM) [(None, 15, 80), (No 51520 tf_op_layer_Sum[0][0]
__________________________________________________________________________________________________
Dropout_GatingOnLSTM (Dropout) (None, 15, 80) 0 history[0][0]
__________________________________________________________________________________________________
gating_act_GatingOnLSTM (Dense) (None, 15, 80) 6480 Dropout_GatingOnLSTM[0][0]
__________________________________________________________________________________________________
gating_GatingOnLSTM (Dense) (None, 15, 80) 6480 Dropout_GatingOnLSTM[0][0]
__________________________________________________________________________________________________
MulGating_GatingOnLSTM (Multipl (None, 15, 80) 0 gating_act_GatingOnLSTM[0][0]
gating_GatingOnLSTM[0][0]
__________________________________________________________________________________________________
add_AfterLSTM (Add) (None, 15, 80) 0 MulGating_GatingOnLSTM[0][0]
tf_op_layer_Sum[0][0]
__________________________________________________________________________________________________
norm_AfterLSTM (LayerNormalizat (None, 15, 80) 160 add_AfterLSTM[0][0]
__________________________________________________________________________________________________
ff_GRN_temp_feature (Dense) (None, 15, 80) 6480 norm_AfterLSTM[0][0]
__________________________________________________________________________________________________
GRN_temp_feature_elu (Activatio (None, 15, 80) 0 ff_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
GRN_temp_feature_LastDense (Den (None, 15, 80) 6480 GRN_temp_feature_elu[0][0]
__________________________________________________________________________________________________
Dropout_GRN_temp_feature (Dropo (None, 15, 80) 0 GRN_temp_feature_LastDense[0][0]
__________________________________________________________________________________________________
gating_act_GRN_temp_feature (De (None, 15, 80) 6480 Dropout_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
gating_GRN_temp_feature (Dense) (None, 15, 80) 6480 Dropout_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
MulGating_GRN_temp_feature (Mul (None, 15, 80) 0 gating_act_GRN_temp_feature[0][0]
gating_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
add_GRN_temp_feature (Add) (None, 15, 80) 0 norm_AfterLSTM[0][0]
MulGating_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
norm_GRN_temp_feature (LayerNor (None, 15, 80) 160 add_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
tf_op_layer_Shape (TensorFlowOp [(3,)] 0 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_16 (T [()] 0 tf_op_layer_Shape[0][0]
__________________________________________________________________________________________________
tf_op_layer_Shape_1 (TensorFlow [(3,)] 0 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
tf_op_layer_eye/Minimum (Tensor [()] 0 tf_op_layer_strided_slice_16[0][0
tf_op_layer_strided_slice_16[0][0
__________________________________________________________________________________________________
dense_2 (Dense) (None, 15, 13) 1040 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_17 (T [(1,)] 0 tf_op_layer_Shape_1[0][0]
__________________________________________________________________________________________________
tf_op_layer_eye/concat/values_1 [(1,)] 0 tf_op_layer_eye/Minimum[0][0]
__________________________________________________________________________________________________
dense_4 (Dense) (None, 15, 13) 1040 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
dense_6 (Dense) (None, 15, 13) 1040 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
dense_8 (Dense) (None, 15, 13) 1040 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
dense_10 (Dense) (None, 15, 13) 1040 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
dense_12 (Dense) (None, 15, 13) 1040 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
tf_op_layer_Shape_2 (TensorFlow [(3,)] 0 dense_2[0][0]
__________________________________________________________________________________________________
tf_op_layer_eye/concat (TensorF [(2,)] 0 tf_op_layer_strided_slice_17[0][0
tf_op_layer_eye/concat/values_1[0
__________________________________________________________________________________________________
tf_op_layer_Shape_3 (TensorFlow [(3,)] 0 dense_4[0][0]
__________________________________________________________________________________________________
tf_op_layer_Shape_4 (TensorFlow [(3,)] 0 dense_6[0][0]
__________________________________________________________________________________________________
tf_op_layer_Shape_5 (TensorFlow [(3,)] 0 dense_8[0][0]
__________________________________________________________________________________________________
tf_op_layer_Shape_6 (TensorFlow [(3,)] 0 dense_10[0][0]
__________________________________________________________________________________________________
tf_op_layer_Shape_7 (TensorFlow [(3,)] 0 dense_12[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_18 (T [()] 0 tf_op_layer_Shape_2[0][0]
__________________________________________________________________________________________________
tf_op_layer_eye/ones (TensorFlo [(None, None)] 0 tf_op_layer_eye/concat[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_19 (T [()] 0 tf_op_layer_Shape_3[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_20 (T [()] 0 tf_op_layer_Shape_4[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_21 (T [()] 0 tf_op_layer_Shape_5[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_22 (T [()] 0 tf_op_layer_Shape_6[0][0]
__________________________________________________________________________________________________
tf_op_layer_strided_slice_23 (T [()] 0 tf_op_layer_Shape_7[0][0]
__________________________________________________________________________________________________
tf_op_layer_Cast (TensorFlowOpL [()] 0 tf_op_layer_strided_slice_18[0][0
__________________________________________________________________________________________________
tf_op_layer_eye/diag (TensorFlo [(None, None, None)] 0 tf_op_layer_eye/ones[0][0]
__________________________________________________________________________________________________
tf_op_layer_Cast_1 (TensorFlowO [()] 0 tf_op_layer_strided_slice_19[0][0
__________________________________________________________________________________________________
tf_op_layer_Cast_2 (TensorFlowO [()] 0 tf_op_layer_strided_slice_20[0][0
__________________________________________________________________________________________________
tf_op_layer_Cast_3 (TensorFlowO [()] 0 tf_op_layer_strided_slice_21[0][0
__________________________________________________________________________________________________
tf_op_layer_Cast_4 (TensorFlowO [()] 0 tf_op_layer_strided_slice_22[0][0
__________________________________________________________________________________________________
tf_op_layer_Cast_5 (TensorFlowO [()] 0 tf_op_layer_strided_slice_23[0][0
__________________________________________________________________________________________________
dense_1 (Dense) (None, 15, 13) 1040 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
tf_op_layer_Sqrt (TensorFlowOpL [()] 0 tf_op_layer_Cast[0][0]
__________________________________________________________________________________________________
tf_op_layer_Cumsum (TensorFlowO [(None, None, None)] 0 tf_op_layer_eye/diag[0][0]
__________________________________________________________________________________________________
dense_3 (Dense) (None, 15, 13) 1040 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
tf_op_layer_Sqrt_1 (TensorFlowO [()] 0 tf_op_layer_Cast_1[0][0]
__________________________________________________________________________________________________
dense_5 (Dense) (None, 15, 13) 1040 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
tf_op_layer_Sqrt_2 (TensorFlowO [()] 0 tf_op_layer_Cast_2[0][0]
__________________________________________________________________________________________________
dense_7 (Dense) (None, 15, 13) 1040 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
tf_op_layer_Sqrt_3 (TensorFlowO [()] 0 tf_op_layer_Cast_3[0][0]
__________________________________________________________________________________________________
dense_9 (Dense) (None, 15, 13) 1040 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
tf_op_layer_Sqrt_4 (TensorFlowO [()] 0 tf_op_layer_Cast_4[0][0]
__________________________________________________________________________________________________
dense_11 (Dense) (None, 15, 13) 1040 norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
tf_op_layer_Sqrt_5 (TensorFlowO [()] 0 tf_op_layer_Cast_5[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenLambda0 (MyLa (None, 15, 15) 0 dense_1[0][0]
dense_2[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenLambdaMask0 ( (None, None, None) 0 tf_op_layer_Cumsum[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenLambda1 (MyLa (None, 15, 15) 0 dense_3[0][0]
dense_4[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenLambdaMask1 ( (None, None, None) 0 tf_op_layer_Cumsum[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenLambda2 (MyLa (None, 15, 15) 0 dense_5[0][0]
dense_6[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenLambdaMask2 ( (None, None, None) 0 tf_op_layer_Cumsum[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenLambda3 (MyLa (None, 15, 15) 0 dense_7[0][0]
dense_8[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenLambdaMask3 ( (None, None, None) 0 tf_op_layer_Cumsum[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenLambda4 (MyLa (None, 15, 15) 0 dense_9[0][0]
dense_10[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenLambdaMask4 ( (None, None, None) 0 tf_op_layer_Cumsum[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenLambda5 (MyLa (None, 15, 15) 0 dense_11[0][0]
dense_12[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenLambdaMask5 ( (None, None, None) 0 tf_op_layer_Cumsum[0][0]
__________________________________________________________________________________________________
SDPA_ADD_0 (Add) (None, 15, 15) 0 ScaledDotProdAttenLambda0[0][0]
ScaledDotProdAttenLambdaMask0[0][
__________________________________________________________________________________________________
SDPA_ADD_1 (Add) (None, 15, 15) 0 ScaledDotProdAttenLambda1[0][0]
ScaledDotProdAttenLambdaMask1[0][
__________________________________________________________________________________________________
SDPA_ADD_2 (Add) (None, 15, 15) 0 ScaledDotProdAttenLambda2[0][0]
ScaledDotProdAttenLambdaMask2[0][
__________________________________________________________________________________________________
SDPA_ADD_3 (Add) (None, 15, 15) 0 ScaledDotProdAttenLambda3[0][0]
ScaledDotProdAttenLambdaMask3[0][
__________________________________________________________________________________________________
SDPA_ADD_4 (Add) (None, 15, 15) 0 ScaledDotProdAttenLambda4[0][0]
ScaledDotProdAttenLambdaMask4[0][
__________________________________________________________________________________________________
SDPA_ADD_5 (Add) (None, 15, 15) 0 ScaledDotProdAttenLambda5[0][0]
ScaledDotProdAttenLambdaMask5[0][
__________________________________________________________________________________________________
ScaledDotProdAtten_softmax (Act (None, 15, 15) 0 SDPA_ADD_0[0][0]
SDPA_ADD_1[0][0]
SDPA_ADD_2[0][0]
SDPA_ADD_3[0][0]
SDPA_ADD_4[0][0]
SDPA_ADD_5[0][0]
__________________________________________________________________________________________________
ScaledDotProdAtten_dropout (Dro (None, 15, 15) 0 ScaledDotProdAtten_softmax[0][0]
ScaledDotProdAtten_softmax[1][0]
ScaledDotProdAtten_softmax[2][0]
ScaledDotProdAtten_softmax[3][0]
ScaledDotProdAtten_softmax[4][0]
ScaledDotProdAtten_softmax[5][0]
__________________________________________________________________________________________________
dense (Dense) (None, 15, 13) 1040 norm_GRN_temp_feature[0][0]
norm_GRN_temp_feature[0][0]
norm_GRN_temp_feature[0][0]
norm_GRN_temp_feature[0][0]
norm_GRN_temp_feature[0][0]
norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenOutput0 (Lamb (None, 15, 13) 0 ScaledDotProdAtten_dropout[0][0]
dense[0][0]
__________________________________________________________________________________________________
ScaledDotProdAttenOutput1 (Lamb (None, 15, 13) 0 ScaledDotProdAtten_dropout[1][0]
dense[1][0]
__________________________________________________________________________________________________
ScaledDotProdAttenOutput2 (Lamb (None, 15, 13) 0 ScaledDotProdAtten_dropout[2][0]
dense[2][0]
__________________________________________________________________________________________________
ScaledDotProdAttenOutput3 (Lamb (None, 15, 13) 0 ScaledDotProdAtten_dropout[3][0]
dense[3][0]
__________________________________________________________________________________________________
ScaledDotProdAttenOutput4 (Lamb (None, 15, 13) 0 ScaledDotProdAtten_dropout[4][0]
dense[4][0]
__________________________________________________________________________________________________
ScaledDotProdAttenOutput5 (Lamb (None, 15, 13) 0 ScaledDotProdAtten_dropout[5][0]
dense[5][0]
__________________________________________________________________________________________________
dropout (Dropout) (None, 15, 13) 0 ScaledDotProdAttenOutput0[0][0]
__________________________________________________________________________________________________
dropout_1 (Dropout) (None, 15, 13) 0 ScaledDotProdAttenOutput1[0][0]
__________________________________________________________________________________________________
dropout_2 (Dropout) (None, 15, 13) 0 ScaledDotProdAttenOutput2[0][0]
__________________________________________________________________________________________________
dropout_3 (Dropout) (None, 15, 13) 0 ScaledDotProdAttenOutput3[0][0]
__________________________________________________________________________________________________
dropout_4 (Dropout) (None, 15, 13) 0 ScaledDotProdAttenOutput4[0][0]
__________________________________________________________________________________________________
dropout_5 (Dropout) (None, 15, 13) 0 ScaledDotProdAttenOutput5[0][0]
__________________________________________________________________________________________________
tf_op_layer_MultiHeadAtten_head [(6, None, 15, 13)] 0 dropout[0][0]
dropout_1[0][0]
dropout_2[0][0]
dropout_3[0][0]
dropout_4[0][0]
dropout_5[0][0]
__________________________________________________________________________________________________
tf_op_layer_Mean (TensorFlowOpL [(None, 15, 13)] 0 tf_op_layer_MultiHeadAtten_heads[
__________________________________________________________________________________________________
MH_atten_output (Dense) (None, 15, 80) 1040 tf_op_layer_Mean[0][0]
__________________________________________________________________________________________________
MHA_output_do (Dropout) (None, 15, 80) 0 MH_atten_output[0][0]
__________________________________________________________________________________________________
Dropout_GatingOnX (Dropout) (None, 15, 80) 0 MHA_output_do[0][0]
__________________________________________________________________________________________________
gating_act_GatingOnX (Dense) (None, 15, 80) 6480 Dropout_GatingOnX[0][0]
__________________________________________________________________________________________________
gating_GatingOnX (Dense) (None, 15, 80) 6480 Dropout_GatingOnX[0][0]
__________________________________________________________________________________________________
MulGating_GatingOnX (Multiply) (None, 15, 80) 0 gating_act_GatingOnX[0][0]
gating_GatingOnX[0][0]
__________________________________________________________________________________________________
add_XAndEnriched (Add) (None, 15, 80) 0 MulGating_GatingOnX[0][0]
norm_GRN_temp_feature[0][0]
__________________________________________________________________________________________________
norm_XAndEnriched (LayerNormali (None, 15, 80) 160 add_XAndEnriched[0][0]
__________________________________________________________________________________________________
ff_NonLinearityOnOut (Dense) (None, 15, 80) 6480 norm_XAndEnriched[0][0]
__________________________________________________________________________________________________
NonLinearityOnOut_elu (Activati (None, 15, 80) 0 ff_NonLinearityOnOut[0][0]
__________________________________________________________________________________________________
NonLinearityOnOut_LastDense (De (None, 15, 80) 6480 NonLinearityOnOut_elu[0][0]
__________________________________________________________________________________________________
Dropout_NonLinearityOnOut (Drop (None, 15, 80) 0 NonLinearityOnOut_LastDense[0][0]
__________________________________________________________________________________________________
gating_act_NonLinearityOnOut (D (None, 15, 80) 6480 Dropout_NonLinearityOnOut[0][0]
__________________________________________________________________________________________________
gating_NonLinearityOnOut (Dense (None, 15, 80) 6480 Dropout_NonLinearityOnOut[0][0]
__________________________________________________________________________________________________
MulGating_NonLinearityOnOut (Mu (None, 15, 80) 0 gating_act_NonLinearityOnOut[0][0
gating_NonLinearityOnOut[0][0]
__________________________________________________________________________________________________
add_NonLinearityOnOut (Add) (None, 15, 80) 0 norm_XAndEnriched[0][0]
MulGating_NonLinearityOnOut[0][0]
__________________________________________________________________________________________________
norm_NonLinearityOnOut (LayerNo (None, 15, 80) 160 add_NonLinearityOnOut[0][0]
__________________________________________________________________________________________________
gating_act_FinalSkip (Dense) (None, 15, 80) 6480 norm_NonLinearityOnOut[0][0]
__________________________________________________________________________________________________
gating_FinalSkip (Dense) (None, 15, 80) 6480 norm_NonLinearityOnOut[0][0]
__________________________________________________________________________________________________
MulGating_FinalSkip (Multiply) (None, 15, 80) 0 gating_act_FinalSkip[0][0]
gating_FinalSkip[0][0]
__________________________________________________________________________________________________
add_DecoderAndTempFeature (Add) (None, 15, 80) 0 MulGating_FinalSkip[0][0]
norm_AfterLSTM[0][0]
__________________________________________________________________________________________________
norm_DecoderAndTempFeature (Lay (None, 15, 80) 160 add_DecoderAndTempFeature[0][0]
__________________________________________________________________________________________________
lambda (Lambda) (None, 80) 0 norm_DecoderAndTempFeature[0][0]
__________________________________________________________________________________________________
Dense_out (Dense) (None, 1) 81 lambda[0][0]
==================================================================================================
Total params: 397,796
Trainable params: 397,796
Non-trainable params: 0
__________________________________________________________________________________________________
"dot" with args ['-Tpng', 'C:\\Users\\USER\\AppData\\Local\\Temp\\tmpxl5tex93'] returned code: 3221225477
stdout, stderr:
b''
b''
dot plot of model could not be plotted due to 3221225477
[9]:
h = model.fit_on_all_training_data(data=data)
***** Training *****
input_x shape: (11854, 15, 7)
target shape: (11854, 1)
***** Validation *****
input_x shape: (2964, 15, 7)
target shape: (2964, 1)
***** Validation *****
input_x shape: (2964, 15, 7)
target shape: (2964, 1)
Train on 14818 samples, validate on 2964 samples
Epoch 1/700
14818/14818 [==============================] - 63s 4ms/sample - loss: 1.1116 - val_loss: 0.8407
Epoch 2/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.7710 - val_loss: 0.6449
Epoch 3/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.6615 - val_loss: 0.6449
Epoch 4/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.5971 - val_loss: 0.4967
Epoch 5/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.5685 - val_loss: 0.4804
Epoch 6/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.5430 - val_loss: 0.4465
Epoch 7/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.5131 - val_loss: 0.5198
Epoch 8/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.5063 - val_loss: 0.4140
Epoch 9/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.4890 - val_loss: 0.5494
Epoch 10/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.4756 - val_loss: 0.4801
Epoch 11/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.4715 - val_loss: 0.3991
Epoch 12/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.4576 - val_loss: 0.4285
Epoch 13/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.4590 - val_loss: 0.3823
Epoch 14/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.4366 - val_loss: 0.3904
Epoch 15/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.4405 - val_loss: 0.3627
Epoch 16/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.4453 - val_loss: 0.3498
Epoch 17/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.4163 - val_loss: 0.3557
Epoch 18/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.4234 - val_loss: 0.3807
Epoch 19/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.4286 - val_loss: 0.3482
Epoch 20/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.4089 - val_loss: 0.3948
Epoch 21/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.3947 - val_loss: 0.3223
Epoch 22/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.3915 - val_loss: 0.3619
Epoch 23/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.3911 - val_loss: 0.3364
Epoch 24/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3941 - val_loss: 0.3572
Epoch 25/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3904 - val_loss: 0.3466
Epoch 26/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3832 - val_loss: 0.3182
Epoch 27/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3758 - val_loss: 0.3313
Epoch 28/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3852 - val_loss: 0.2972ETA: 0s - loss: 0.38
Epoch 29/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.3843 - val_loss: 0.3558
Epoch 30/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3768 - val_loss: 0.3087
Epoch 31/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3708 - val_loss: 0.3024
Epoch 32/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3646 - val_loss: 0.2874
Epoch 33/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3619 - val_loss: 0.3081
Epoch 34/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3587 - val_loss: 0.2820
Epoch 35/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3547 - val_loss: 0.2996
Epoch 36/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3484 - val_loss: 0.3026
Epoch 37/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3456 - val_loss: 0.2890
Epoch 38/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3347 - val_loss: 0.2922
Epoch 39/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3431 - val_loss: 0.2783
Epoch 40/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3406 - val_loss: 0.2889
Epoch 41/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3369 - val_loss: 0.2809
Epoch 42/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3363 - val_loss: 0.2752
Epoch 43/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3350 - val_loss: 0.2763
Epoch 44/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3293 - val_loss: 0.2828
Epoch 45/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.3198 - val_loss: 0.2971
Epoch 46/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3188 - val_loss: 0.2794
Epoch 47/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.3212 - val_loss: 0.2641
Epoch 48/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3200 - val_loss: 0.2783
Epoch 49/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3125 - val_loss: 0.3119
Epoch 50/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3033 - val_loss: 0.3003
Epoch 51/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.3017 - val_loss: 0.2929
Epoch 52/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.3141 - val_loss: 0.2583
Epoch 53/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3097 - val_loss: 0.2696
Epoch 54/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.3010 - val_loss: 0.3130
Epoch 55/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.3065 - val_loss: 0.2532
Epoch 56/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2932 - val_loss: 0.2528
Epoch 57/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2962 - val_loss: 0.2481
Epoch 58/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2856 - val_loss: 0.2617
Epoch 59/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2905 - val_loss: 0.2447
Epoch 60/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2949 - val_loss: 0.2430
Epoch 61/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2899 - val_loss: 0.2416
Epoch 62/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.2884 - val_loss: 0.2415
Epoch 63/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2860 - val_loss: 0.2355
Epoch 64/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2830 - val_loss: 0.2401
Epoch 65/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2800 - val_loss: 0.2332
Epoch 66/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2719 - val_loss: 0.2323
Epoch 67/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2759 - val_loss: 0.2555
Epoch 68/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2819 - val_loss: 0.2567
Epoch 69/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2738 - val_loss: 0.2396
Epoch 70/700
14818/14818 [==============================] - 44s 3ms/sample - loss: 0.2719 - val_loss: 0.2411
Epoch 71/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.2666 - val_loss: 0.2413
Epoch 72/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2634 - val_loss: 0.2683
Epoch 73/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2599 - val_loss: 0.2316
Epoch 74/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2630 - val_loss: 0.2268
Epoch 75/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2671 - val_loss: 0.2422
Epoch 76/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2615 - val_loss: 0.2113
Epoch 77/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.2597 - val_loss: 0.2234
Epoch 78/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2488 - val_loss: 0.2274
Epoch 79/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.2581 - val_loss: 0.2240
Epoch 80/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2521 - val_loss: 0.2803
Epoch 81/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2479 - val_loss: 0.2136
Epoch 82/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2499 - val_loss: 0.2320
Epoch 83/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2499 - val_loss: 0.2033
Epoch 84/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.2497 - val_loss: 0.2104
Epoch 85/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2448 - val_loss: 0.2562
Epoch 86/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2396 - val_loss: 0.2089
Epoch 87/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2395 - val_loss: 0.2039
Epoch 88/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2444 - val_loss: 0.2073
Epoch 89/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2440 - val_loss: 0.2011
Epoch 90/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2388 - val_loss: 0.2152
Epoch 91/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2268 - val_loss: 0.1937
Epoch 92/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2325 - val_loss: 0.1942
Epoch 93/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2322 - val_loss: 0.1928
Epoch 94/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2286 - val_loss: 0.1971
Epoch 95/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2372 - val_loss: 0.1909
Epoch 96/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2296 - val_loss: 0.2054
Epoch 97/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2218 - val_loss: 0.2055
Epoch 98/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2243 - val_loss: 0.1972
Epoch 99/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2170 - val_loss: 0.1960
Epoch 100/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2191 - val_loss: 0.2004
Epoch 101/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2209 - val_loss: 0.2011
Epoch 102/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2167 - val_loss: 0.1961
Epoch 103/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2179 - val_loss: 0.1951
Epoch 104/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2178 - val_loss: 0.2021
Epoch 105/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.2196 - val_loss: 0.1871
Epoch 106/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.2148 - val_loss: 0.1942
Epoch 107/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2112 - val_loss: 0.2451
Epoch 108/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2136 - val_loss: 0.1863
Epoch 109/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2080 - val_loss: 0.1801
Epoch 110/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.2012 - val_loss: 0.1912
Epoch 111/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2097 - val_loss: 0.1771
Epoch 112/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2048 - val_loss: 0.1796
Epoch 113/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1995 - val_loss: 0.1807
Epoch 114/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1950 - val_loss: 0.2304
Epoch 115/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.2041 - val_loss: 0.1823
Epoch 116/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.2007 - val_loss: 0.1762
Epoch 117/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1957 - val_loss: 0.1660
Epoch 118/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1921 - val_loss: 0.1684
Epoch 119/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1922 - val_loss: 0.1694
Epoch 120/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1929 - val_loss: 0.1748
Epoch 121/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1947 - val_loss: 0.1616
Epoch 122/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1919 - val_loss: 0.1779
Epoch 123/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1841 - val_loss: 0.1783
Epoch 124/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1821 - val_loss: 0.1720
Epoch 125/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1853 - val_loss: 0.1708
Epoch 126/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1810 - val_loss: 0.1612
Epoch 127/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1801 - val_loss: 0.1716
Epoch 128/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1892 - val_loss: 0.1600
Epoch 129/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1832 - val_loss: 0.1592
Epoch 130/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1757 - val_loss: 0.1622
Epoch 131/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1783 - val_loss: 0.1560
Epoch 132/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1734 - val_loss: 0.1570
Epoch 133/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1730 - val_loss: 0.1535
Epoch 134/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1781 - val_loss: 0.1574
Epoch 135/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1709 - val_loss: 0.1733
Epoch 136/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1732 - val_loss: 0.1552
Epoch 137/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1747 - val_loss: 0.1575
Epoch 138/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1698 - val_loss: 0.1551
Epoch 139/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1694 - val_loss: 0.1541
Epoch 140/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1689 - val_loss: 0.1501
Epoch 141/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1619 - val_loss: 0.1711
Epoch 142/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1652 - val_loss: 0.1534
Epoch 143/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1672 - val_loss: 0.1523
Epoch 144/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1676 - val_loss: 0.1486
Epoch 145/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1635 - val_loss: 0.1552
Epoch 146/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1581 - val_loss: 0.1523
Epoch 147/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1656 - val_loss: 0.1518
Epoch 148/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1591 - val_loss: 0.1399
Epoch 149/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1542 - val_loss: 0.1513
Epoch 150/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1609 - val_loss: 0.1630
Epoch 151/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1533 - val_loss: 0.1461
Epoch 152/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1601 - val_loss: 0.1557
Epoch 153/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1551 - val_loss: 0.1346
Epoch 154/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1522 - val_loss: 0.1491
Epoch 155/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1502 - val_loss: 0.1351
Epoch 156/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1550 - val_loss: 0.1400
Epoch 157/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1561 - val_loss: 0.1381
Epoch 158/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1472 - val_loss: 0.1393
Epoch 159/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1547 - val_loss: 0.1299
Epoch 160/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1557 - val_loss: 0.1317
Epoch 161/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1478 - val_loss: 0.1355
Epoch 162/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1481 - val_loss: 0.1339
Epoch 163/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1448 - val_loss: 0.1438
Epoch 164/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1478 - val_loss: 0.1292
Epoch 165/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1470 - val_loss: 0.1413
Epoch 166/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1450 - val_loss: 0.1248
Epoch 167/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1432 - val_loss: 0.1269
Epoch 168/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1437 - val_loss: 0.1294
Epoch 169/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1430 - val_loss: 0.1487
Epoch 170/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1398 - val_loss: 0.1311
Epoch 171/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1436 - val_loss: 0.1231
Epoch 172/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1388 - val_loss: 0.1347
Epoch 173/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1391 - val_loss: 0.1178
Epoch 174/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1393 - val_loss: 0.1353
Epoch 175/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1386 - val_loss: 0.1285
Epoch 176/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1370 - val_loss: 0.1260
Epoch 177/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1365 - val_loss: 0.1266
Epoch 178/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1345 - val_loss: 0.1254
Epoch 179/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1343 - val_loss: 0.1188
Epoch 180/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1279 - val_loss: 0.1170
Epoch 181/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.1328 - val_loss: 0.1516
Epoch 182/700
14818/14818 [==============================] - 49s 3ms/sample - loss: 0.1376 - val_loss: 0.1159
Epoch 183/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1308 - val_loss: 0.1193
Epoch 184/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1310 - val_loss: 0.1152
Epoch 185/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1346 - val_loss: 0.1191
Epoch 186/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1297 - val_loss: 0.1239
Epoch 187/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1290 - val_loss: 0.1143
Epoch 188/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1273 - val_loss: 0.1229
Epoch 189/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1259 - val_loss: 0.1081
Epoch 190/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1305 - val_loss: 0.1115
Epoch 191/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1234 - val_loss: 0.1119
Epoch 192/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1219 - val_loss: 0.1085
Epoch 193/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1250 - val_loss: 0.1171
Epoch 194/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1228 - val_loss: 0.1217
Epoch 195/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1231 - val_loss: 0.1102
Epoch 196/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1237 - val_loss: 0.1090
Epoch 197/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1223 - val_loss: 0.1358
Epoch 198/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1220 - val_loss: 0.1047
Epoch 199/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1246 - val_loss: 0.1115
Epoch 200/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1189 - val_loss: 0.1037
Epoch 201/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1212 - val_loss: 0.1048
Epoch 202/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1227 - val_loss: 0.1020
Epoch 203/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1199 - val_loss: 0.1123
Epoch 204/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1165 - val_loss: 0.1091
Epoch 205/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1164 - val_loss: 0.1191
Epoch 206/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1182 - val_loss: 0.0993
Epoch 207/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1168 - val_loss: 0.1030
Epoch 208/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1147 - val_loss: 0.0983
Epoch 209/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1145 - val_loss: 0.1014
Epoch 210/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1135 - val_loss: 0.0992
Epoch 211/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1195 - val_loss: 0.0985
Epoch 212/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1111 - val_loss: 0.1062
Epoch 213/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1175 - val_loss: 0.0972
Epoch 214/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1119 - val_loss: 0.1049
Epoch 215/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1109 - val_loss: 0.0977
Epoch 216/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1138 - val_loss: 0.1036
Epoch 217/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1115 - val_loss: 0.1116
Epoch 218/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1165 - val_loss: 0.0965
Epoch 219/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1079 - val_loss: 0.0916
Epoch 220/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1074 - val_loss: 0.0959
Epoch 221/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1118 - val_loss: 0.1023
Epoch 222/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.1062 - val_loss: 0.0990
Epoch 223/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1086 - val_loss: 0.0892
Epoch 224/700
14818/14818 [==============================] - 49s 3ms/sample - loss: 0.1109 - val_loss: 0.0886
Epoch 225/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1065 - val_loss: 0.0974
Epoch 226/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1057 - val_loss: 0.0891
Epoch 227/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1035 - val_loss: 0.0973
Epoch 228/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1046 - val_loss: 0.1087
Epoch 229/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1087 - val_loss: 0.0892
Epoch 230/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1095 - val_loss: 0.1075
Epoch 231/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1034 - val_loss: 0.0909
Epoch 232/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1039 - val_loss: 0.0902
Epoch 233/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1058 - val_loss: 0.0932
Epoch 234/700
14818/14818 [==============================] - 49s 3ms/sample - loss: 0.1026 - val_loss: 0.0875
Epoch 235/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1033 - val_loss: 0.0850
Epoch 236/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1008 - val_loss: 0.0842
Epoch 237/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1008 - val_loss: 0.0889
Epoch 238/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1009 - val_loss: 0.0873
Epoch 239/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1015 - val_loss: 0.0960
Epoch 240/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0997 - val_loss: 0.0988
Epoch 241/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0978 - val_loss: 0.0872
Epoch 242/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0977 - val_loss: 0.0869
Epoch 243/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0968 - val_loss: 0.0832
Epoch 244/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0979 - val_loss: 0.0870
Epoch 245/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1005 - val_loss: 0.0879
Epoch 246/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0987 - val_loss: 0.0810
Epoch 247/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.1005 - val_loss: 0.0813
Epoch 248/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0968 - val_loss: 0.0873
Epoch 249/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0978 - val_loss: 0.0813
Epoch 250/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0951 - val_loss: 0.0858
Epoch 251/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0973 - val_loss: 0.0810
Epoch 252/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0937 - val_loss: 0.0823
Epoch 253/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0946 - val_loss: 0.0743
Epoch 254/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0960 - val_loss: 0.0805
Epoch 255/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0962 - val_loss: 0.0884
Epoch 256/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0917 - val_loss: 0.0773
Epoch 257/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0939 - val_loss: 0.0779
Epoch 258/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0907 - val_loss: 0.0746
Epoch 259/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0930 - val_loss: 0.0756
Epoch 260/700
14818/14818 [==============================] - 44s 3ms/sample - loss: 0.0908 - val_loss: 0.0867
Epoch 261/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0938 - val_loss: 0.0750
Epoch 262/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0903 - val_loss: 0.0724
Epoch 263/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0915 - val_loss: 0.0800
Epoch 264/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0901 - val_loss: 0.0735
Epoch 265/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0918 - val_loss: 0.0756
Epoch 266/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0905 - val_loss: 0.0779
Epoch 267/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0889 - val_loss: 0.0687
Epoch 268/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0897 - val_loss: 0.0787
Epoch 269/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0857 - val_loss: 0.0751
Epoch 270/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0916 - val_loss: 0.0721
Epoch 271/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0882 - val_loss: 0.0792
Epoch 272/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0904 - val_loss: 0.0879
Epoch 273/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0882 - val_loss: 0.0744
Epoch 274/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0869 - val_loss: 0.0728
Epoch 275/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0876 - val_loss: 0.0757
Epoch 276/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0871 - val_loss: 0.0758
Epoch 277/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0856 - val_loss: 0.0734
Epoch 278/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0869 - val_loss: 0.0685
Epoch 279/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0857 - val_loss: 0.0664
Epoch 280/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0837 - val_loss: 0.0701
Epoch 281/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0888 - val_loss: 0.0730
Epoch 282/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0833 - val_loss: 0.0744
Epoch 283/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0863 - val_loss: 0.0678
Epoch 284/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0833 - val_loss: 0.0717
Epoch 285/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0836 - val_loss: 0.0686
Epoch 286/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0827 - val_loss: 0.0721
Epoch 287/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0810 - val_loss: 0.0644
Epoch 288/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0824 - val_loss: 0.0650
Epoch 289/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0825 - val_loss: 0.0708
Epoch 290/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0825 - val_loss: 0.0685
Epoch 291/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0791 - val_loss: 0.0740
Epoch 292/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0826 - val_loss: 0.0636
Epoch 293/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0811 - val_loss: 0.0726
Epoch 294/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0801 - val_loss: 0.0618
Epoch 295/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0818 - val_loss: 0.0759
Epoch 296/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0792 - val_loss: 0.0656
Epoch 297/700
14818/14818 [==============================] - 44s 3ms/sample - loss: 0.0815 - val_loss: 0.0615
Epoch 298/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0767 - val_loss: 0.0658
Epoch 299/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0816 - val_loss: 0.0649
Epoch 300/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0797 - val_loss: 0.0608
Epoch 301/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0788 - val_loss: 0.0622
Epoch 302/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0783 - val_loss: 0.0607
Epoch 303/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0807 - val_loss: 0.0687
Epoch 304/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0780 - val_loss: 0.0621
Epoch 305/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0772 - val_loss: 0.0608
Epoch 306/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0783 - val_loss: 0.0590
Epoch 307/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0777 - val_loss: 0.0592
Epoch 308/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0755 - val_loss: 0.0629
Epoch 309/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0779 - val_loss: 0.0717
Epoch 310/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0754 - val_loss: 0.0643
Epoch 311/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0762 - val_loss: 0.0698
Epoch 312/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0754 - val_loss: 0.0619
Epoch 313/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0743 - val_loss: 0.0591
Epoch 314/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0752 - val_loss: 0.0679
Epoch 315/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0757 - val_loss: 0.0590
Epoch 316/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0769 - val_loss: 0.0609
Epoch 317/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0771 - val_loss: 0.0579
Epoch 318/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0733 - val_loss: 0.0565
Epoch 319/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0727 - val_loss: 0.0562
Epoch 320/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0733 - val_loss: 0.0567
Epoch 321/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0731 - val_loss: 0.0591
Epoch 322/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0741 - val_loss: 0.0599
Epoch 323/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0722 - val_loss: 0.0556
Epoch 324/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0725 - val_loss: 0.0579
Epoch 325/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0711 - val_loss: 0.0535
Epoch 326/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0722 - val_loss: 0.0661
Epoch 327/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0714 - val_loss: 0.0536
Epoch 328/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0709 - val_loss: 0.0566
Epoch 329/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0733 - val_loss: 0.0550
Epoch 330/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0711 - val_loss: 0.0562
Epoch 331/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0703 - val_loss: 0.0604
Epoch 332/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0716 - val_loss: 0.0534
Epoch 333/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0713 - val_loss: 0.0572
Epoch 334/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0709 - val_loss: 0.0590
Epoch 335/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0692 - val_loss: 0.0570
Epoch 336/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0701 - val_loss: 0.0535
Epoch 337/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0702 - val_loss: 0.0572
Epoch 338/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0698 - val_loss: 0.0529
Epoch 339/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0703 - val_loss: 0.0597
Epoch 340/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0691 - val_loss: 0.0543
Epoch 341/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0693 - val_loss: 0.0494
Epoch 342/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0689 - val_loss: 0.0540
Epoch 343/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0672 - val_loss: 0.0543
Epoch 344/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0683 - val_loss: 0.0530
Epoch 345/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0670 - val_loss: 0.0553
Epoch 346/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0653 - val_loss: 0.0500
Epoch 347/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0676 - val_loss: 0.0502
Epoch 348/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0659 - val_loss: 0.0538
Epoch 349/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0676 - val_loss: 0.0518
Epoch 350/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0662 - val_loss: 0.0525
Epoch 351/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0660 - val_loss: 0.0492
Epoch 352/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0684 - val_loss: 0.0502
Epoch 353/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0654 - val_loss: 0.0500
Epoch 354/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0646 - val_loss: 0.0518
Epoch 355/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0658 - val_loss: 0.0581
Epoch 356/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0663 - val_loss: 0.0574
Epoch 357/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0665 - val_loss: 0.0510
Epoch 358/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0646 - val_loss: 0.0484
Epoch 359/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0637 - val_loss: 0.0509
Epoch 360/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0629 - val_loss: 0.0567
Epoch 361/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0635 - val_loss: 0.0518
Epoch 362/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0635 - val_loss: 0.0547
Epoch 363/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0642 - val_loss: 0.0475
Epoch 364/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0636 - val_loss: 0.0476
Epoch 365/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0612 - val_loss: 0.0450
Epoch 366/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0629 - val_loss: 0.0478
Epoch 367/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0648 - val_loss: 0.0475
Epoch 368/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0625 - val_loss: 0.0512
Epoch 369/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0613 - val_loss: 0.0470
Epoch 370/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0625 - val_loss: 0.0489
Epoch 371/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0600 - val_loss: 0.0468
Epoch 372/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0603 - val_loss: 0.0480
Epoch 373/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0622 - val_loss: 0.0463
Epoch 374/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0598 - val_loss: 0.0557
Epoch 375/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0634 - val_loss: 0.0447
Epoch 376/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0610 - val_loss: 0.0460
Epoch 377/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0624 - val_loss: 0.0566
Epoch 378/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0603 - val_loss: 0.0459
Epoch 379/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0602 - val_loss: 0.0439
Epoch 380/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0595 - val_loss: 0.0443
Epoch 381/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0595 - val_loss: 0.0430
Epoch 382/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0593 - val_loss: 0.0499
Epoch 383/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0584 - val_loss: 0.0423
Epoch 384/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0579 - val_loss: 0.0418
Epoch 385/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0586 - val_loss: 0.0466
Epoch 386/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0587 - val_loss: 0.0447
Epoch 387/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0624 - val_loss: 0.0427
Epoch 388/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0568 - val_loss: 0.0442
Epoch 389/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0556 - val_loss: 0.0419
Epoch 390/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0583 - val_loss: 0.0425
Epoch 391/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0578 - val_loss: 0.0440
Epoch 392/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0566 - val_loss: 0.0421
Epoch 393/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0572 - val_loss: 0.0414
Epoch 394/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0565 - val_loss: 0.0406
Epoch 395/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0562 - val_loss: 0.0425
Epoch 396/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0559 - val_loss: 0.0401
Epoch 397/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0565 - val_loss: 0.0413
Epoch 398/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0567 - val_loss: 0.0430
Epoch 399/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0548 - val_loss: 0.0446
Epoch 400/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0565 - val_loss: 0.0453
Epoch 401/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0564 - val_loss: 0.0393
Epoch 402/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0548 - val_loss: 0.0472
Epoch 403/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0578 - val_loss: 0.0420
Epoch 404/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0521 - val_loss: 0.0391
Epoch 405/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0573 - val_loss: 0.0431
Epoch 406/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0547 - val_loss: 0.0450
Epoch 407/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0569 - val_loss: 0.0406
Epoch 408/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0532 - val_loss: 0.0377
Epoch 409/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0536 - val_loss: 0.0391
Epoch 410/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0548 - val_loss: 0.0387
Epoch 411/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0533 - val_loss: 0.0410
Epoch 412/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0535 - val_loss: 0.0394
Epoch 413/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0541 - val_loss: 0.0392
Epoch 414/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0522 - val_loss: 0.0378
Epoch 415/700
14818/14818 [==============================] - 42s 3ms/sample - loss: 0.0529 - val_loss: 0.0379
Epoch 416/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0534 - val_loss: 0.0382
Epoch 417/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0551 - val_loss: 0.0399
Epoch 418/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0527 - val_loss: 0.0402
Epoch 419/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0540 - val_loss: 0.0441
Epoch 420/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0534 - val_loss: 0.0355
Epoch 421/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0515 - val_loss: 0.0363
Epoch 422/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0521 - val_loss: 0.0384
Epoch 423/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0540 - val_loss: 0.0375
Epoch 424/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0516 - val_loss: 0.0350
Epoch 425/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0512 - val_loss: 0.0387
Epoch 426/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0520 - val_loss: 0.0437
Epoch 427/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0512 - val_loss: 0.0380
Epoch 428/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0513 - val_loss: 0.0372
Epoch 429/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0506 - val_loss: 0.0361
Epoch 430/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0503 - val_loss: 0.0398
Epoch 431/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0498 - val_loss: 0.0355
Epoch 432/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0511 - val_loss: 0.0355
Epoch 433/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0522 - val_loss: 0.0372
Epoch 434/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0506 - val_loss: 0.0363
Epoch 435/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0481 - val_loss: 0.0362
Epoch 436/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0506 - val_loss: 0.0358
Epoch 437/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0499 - val_loss: 0.0363
Epoch 438/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0509 - val_loss: 0.0366
Epoch 439/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0510 - val_loss: 0.0368
Epoch 440/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0495 - val_loss: 0.0418
Epoch 441/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0498 - val_loss: 0.0344
Epoch 442/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0485 - val_loss: 0.0387
Epoch 443/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0498 - val_loss: 0.0355
Epoch 444/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0494 - val_loss: 0.0339
Epoch 445/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0477 - val_loss: 0.0327
Epoch 446/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0470 - val_loss: 0.0326
Epoch 447/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0482 - val_loss: 0.0451
Epoch 448/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0483 - val_loss: 0.0323
Epoch 449/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0485 - val_loss: 0.0371
Epoch 450/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0480 - val_loss: 0.0379
Epoch 451/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0479 - val_loss: 0.0327
Epoch 452/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0466 - val_loss: 0.0352
Epoch 453/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0480 - val_loss: 0.0333
Epoch 454/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0473 - val_loss: 0.0344
Epoch 455/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0483 - val_loss: 0.0400
Epoch 456/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0451 - val_loss: 0.0342
Epoch 457/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0478 - val_loss: 0.0344
Epoch 458/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0471 - val_loss: 0.0379
Epoch 459/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0476 - val_loss: 0.0364
Epoch 460/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0462 - val_loss: 0.0330
Epoch 461/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0448 - val_loss: 0.0313
Epoch 462/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0455 - val_loss: 0.0320
Epoch 463/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0464 - val_loss: 0.0334
Epoch 464/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0465 - val_loss: 0.0321
Epoch 465/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0462 - val_loss: 0.0326
Epoch 466/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0473 - val_loss: 0.0358
Epoch 467/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0450 - val_loss: 0.0357
Epoch 468/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0440 - val_loss: 0.0365
Epoch 469/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0450 - val_loss: 0.0320
Epoch 470/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0453 - val_loss: 0.0311
Epoch 471/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0458 - val_loss: 0.0350
Epoch 472/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0446 - val_loss: 0.0357
Epoch 473/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0447 - val_loss: 0.0330
Epoch 474/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0444 - val_loss: 0.0299
Epoch 475/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0446 - val_loss: 0.0323
Epoch 476/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0432 - val_loss: 0.0326
Epoch 477/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0441 - val_loss: 0.0367
Epoch 478/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0458 - val_loss: 0.0348
Epoch 479/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0442 - val_loss: 0.0364
Epoch 480/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0442 - val_loss: 0.0341
Epoch 481/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0436 - val_loss: 0.0384
Epoch 482/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0435 - val_loss: 0.0288
Epoch 483/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0419 - val_loss: 0.0336
Epoch 484/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0443 - val_loss: 0.0347
Epoch 485/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0427 - val_loss: 0.0327
Epoch 486/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0433 - val_loss: 0.0291
Epoch 487/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0413 - val_loss: 0.0335
Epoch 488/700
14818/14818 [==============================] - 44s 3ms/sample - loss: 0.0440 - val_loss: 0.0292
Epoch 489/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0424 - val_loss: 0.0336
Epoch 490/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0417 - val_loss: 0.0314
Epoch 491/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0428 - val_loss: 0.0308
Epoch 492/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0423 - val_loss: 0.0297
Epoch 493/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0425 - val_loss: 0.0299
Epoch 494/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0421 - val_loss: 0.0334
Epoch 495/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0419 - val_loss: 0.0331
Epoch 496/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0415 - val_loss: 0.0295
Epoch 497/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0406 - val_loss: 0.0281
Epoch 498/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0418 - val_loss: 0.0291
Epoch 499/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0409 - val_loss: 0.0279
Epoch 500/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0409 - val_loss: 0.0295
Epoch 501/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0407 - val_loss: 0.0279
Epoch 502/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0417 - val_loss: 0.0304
Epoch 503/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0395 - val_loss: 0.0274
Epoch 504/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0398 - val_loss: 0.0330
Epoch 505/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0402 - val_loss: 0.0268
Epoch 506/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0404 - val_loss: 0.0319
Epoch 507/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0400 - val_loss: 0.0279
Epoch 508/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0411 - val_loss: 0.0276
Epoch 509/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0395 - val_loss: 0.0265
Epoch 510/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0396 - val_loss: 0.0293
Epoch 511/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0409 - val_loss: 0.0288
Epoch 512/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0395 - val_loss: 0.0278
Epoch 513/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0396 - val_loss: 0.0266
Epoch 514/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0405 - val_loss: 0.0285
Epoch 515/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0390 - val_loss: 0.0267
Epoch 516/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0393 - val_loss: 0.0295
Epoch 517/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0390 - val_loss: 0.0267
Epoch 518/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0387 - val_loss: 0.0294
Epoch 519/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0406 - val_loss: 0.0260
Epoch 520/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0387 - val_loss: 0.0261
Epoch 521/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0387 - val_loss: 0.0259
Epoch 522/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0391 - val_loss: 0.0276
Epoch 523/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0390 - val_loss: 0.0290
Epoch 524/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0385 - val_loss: 0.0263
Epoch 525/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0386 - val_loss: 0.0267
Epoch 526/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0380 - val_loss: 0.0320
Epoch 527/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0372 - val_loss: 0.0267
Epoch 528/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0382 - val_loss: 0.0270
Epoch 529/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0397 - val_loss: 0.0267
Epoch 530/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0383 - val_loss: 0.0261
Epoch 531/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0372 - val_loss: 0.0276
Epoch 532/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0377 - val_loss: 0.0285
Epoch 533/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0372 - val_loss: 0.0267
Epoch 534/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0378 - val_loss: 0.0275
Epoch 535/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0373 - val_loss: 0.0246
Epoch 536/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0388 - val_loss: 0.0267
Epoch 537/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0370 - val_loss: 0.0263
Epoch 538/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0385 - val_loss: 0.0283
Epoch 539/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0369 - val_loss: 0.0247
Epoch 540/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0363 - val_loss: 0.0261
Epoch 541/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0362 - val_loss: 0.0265
Epoch 542/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0358 - val_loss: 0.0286
Epoch 543/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0358 - val_loss: 0.0288
Epoch 544/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0370 - val_loss: 0.0256
Epoch 545/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0365 - val_loss: 0.0267
Epoch 546/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0380 - val_loss: 0.0245
Epoch 547/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0352 - val_loss: 0.0265
Epoch 548/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0371 - val_loss: 0.0258
Epoch 549/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0361 - val_loss: 0.0270
Epoch 550/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0352 - val_loss: 0.0260
Epoch 551/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0349 - val_loss: 0.0270
Epoch 552/700
14818/14818 [==============================] - 44s 3ms/sample - loss: 0.0358 - val_loss: 0.0259
Epoch 553/700
14818/14818 [==============================] - 43s 3ms/sample - loss: 0.0366 - val_loss: 0.0267
Epoch 554/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0362 - val_loss: 0.0247
Epoch 555/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0351 - val_loss: 0.0238
Epoch 556/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0349 - val_loss: 0.0241
Epoch 557/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0352 - val_loss: 0.0264
Epoch 558/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0347 - val_loss: 0.0264
Epoch 559/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0356 - val_loss: 0.0223
Epoch 560/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0354 - val_loss: 0.0258
Epoch 561/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0355 - val_loss: 0.0242
Epoch 562/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0333 - val_loss: 0.0236
Epoch 563/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0344 - val_loss: 0.0246
Epoch 564/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0342 - val_loss: 0.0230
Epoch 565/700
14818/14818 [==============================] - 44s 3ms/sample - loss: 0.0342 - val_loss: 0.0256
Epoch 566/700
14818/14818 [==============================] - 43s 3ms/sample - loss: 0.0342 - val_loss: 0.0241
Epoch 567/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0340 - val_loss: 0.0225
Epoch 568/700
14818/14818 [==============================] - 44s 3ms/sample - loss: 0.0344 - val_loss: 0.0267
Epoch 569/700
14818/14818 [==============================] - 44s 3ms/sample - loss: 0.0343 - val_loss: 0.0263
Epoch 570/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0347 - val_loss: 0.0231
Epoch 571/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0339 - val_loss: 0.0306
Epoch 572/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0344 - val_loss: 0.0250
Epoch 573/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0341 - val_loss: 0.0219
Epoch 574/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0335 - val_loss: 0.0248
Epoch 575/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0344 - val_loss: 0.0215
Epoch 576/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0339 - val_loss: 0.0239
Epoch 577/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0336 - val_loss: 0.0246
Epoch 578/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0334 - val_loss: 0.0215
Epoch 579/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0329 - val_loss: 0.0232
Epoch 580/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0337 - val_loss: 0.0221
Epoch 581/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0344 - val_loss: 0.0224
Epoch 582/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0326 - val_loss: 0.0226
Epoch 583/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0331 - val_loss: 0.0240
Epoch 584/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0323 - val_loss: 0.0263
Epoch 585/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0341 - val_loss: 0.0245
Epoch 586/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0318 - val_loss: 0.0210
Epoch 587/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0319 - val_loss: 0.0226
Epoch 588/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0320 - val_loss: 0.0258
Epoch 589/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0319 - val_loss: 0.0273
Epoch 590/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0332 - val_loss: 0.0222
Epoch 591/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0324 - val_loss: 0.0213
Epoch 592/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0319 - val_loss: 0.0219
Epoch 593/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0309 - val_loss: 0.0222
Epoch 594/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0319 - val_loss: 0.0214
Epoch 595/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0332 - val_loss: 0.0244
Epoch 596/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0326 - val_loss: 0.0245
Epoch 597/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0317 - val_loss: 0.0212
Epoch 598/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0320 - val_loss: 0.0233
Epoch 599/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0312 - val_loss: 0.0204
Epoch 600/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0318 - val_loss: 0.0238
Epoch 601/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0312 - val_loss: 0.0234
Epoch 602/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0321 - val_loss: 0.0241
Epoch 603/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0321 - val_loss: 0.0221
Epoch 604/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0314 - val_loss: 0.0268
Epoch 605/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0312 - val_loss: 0.0218
Epoch 606/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0309 - val_loss: 0.0201
Epoch 607/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0309 - val_loss: 0.0201
Epoch 608/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0303 - val_loss: 0.0216
Epoch 609/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0310 - val_loss: 0.0201
Epoch 610/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0300 - val_loss: 0.0210
Epoch 611/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0307 - val_loss: 0.0228
Epoch 612/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0315 - val_loss: 0.0206
Epoch 613/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0297 - val_loss: 0.0207
Epoch 614/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0307 - val_loss: 0.0198
Epoch 615/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0303 - val_loss: 0.0239
Epoch 616/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0300 - val_loss: 0.0220
Epoch 617/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0311 - val_loss: 0.0198
Epoch 618/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0306 - val_loss: 0.0202
Epoch 619/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0299 - val_loss: 0.0201
Epoch 620/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0293 - val_loss: 0.0198
Epoch 621/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0299 - val_loss: 0.0232
Epoch 622/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0301 - val_loss: 0.0208
Epoch 623/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0295 - val_loss: 0.0193
Epoch 624/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0295 - val_loss: 0.0205
Epoch 625/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0297 - val_loss: 0.0228
Epoch 626/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0289 - val_loss: 0.0203
Epoch 627/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0290 - val_loss: 0.0192
Epoch 628/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0292 - val_loss: 0.0197
Epoch 629/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0290 - val_loss: 0.0215
Epoch 630/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0297 - val_loss: 0.0202
Epoch 631/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0299 - val_loss: 0.0197
Epoch 632/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0296 - val_loss: 0.0186
Epoch 633/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0288 - val_loss: 0.0187
Epoch 634/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0289 - val_loss: 0.0203
Epoch 635/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0288 - val_loss: 0.0180
Epoch 636/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0281 - val_loss: 0.0185
Epoch 637/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0280 - val_loss: 0.0191
Epoch 638/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0283 - val_loss: 0.0205
Epoch 639/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0290 - val_loss: 0.0190
Epoch 640/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0274 - val_loss: 0.0187
Epoch 641/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0288 - val_loss: 0.0207
Epoch 642/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0297 - val_loss: 0.0183
Epoch 643/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0281 - val_loss: 0.0195
Epoch 644/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0277 - val_loss: 0.0193
Epoch 645/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0281 - val_loss: 0.0192
Epoch 646/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0271 - val_loss: 0.0188
Epoch 647/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0287 - val_loss: 0.0190
Epoch 648/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0279 - val_loss: 0.0200
Epoch 649/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0293 - val_loss: 0.0261
Epoch 650/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0278 - val_loss: 0.0194
Epoch 651/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0276 - val_loss: 0.0189
Epoch 652/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0273 - val_loss: 0.0182
Epoch 653/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0277 - val_loss: 0.0173
Epoch 654/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0275 - val_loss: 0.0215
Epoch 655/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0267 - val_loss: 0.0192
Epoch 656/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0274 - val_loss: 0.0181
Epoch 657/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0275 - val_loss: 0.0192
Epoch 658/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0269 - val_loss: 0.0175
Epoch 659/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0281 - val_loss: 0.0179
Epoch 660/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0257 - val_loss: 0.0173
Epoch 661/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0260 - val_loss: 0.0185
Epoch 662/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0266 - val_loss: 0.0238
Epoch 663/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0265 - val_loss: 0.0166
Epoch 664/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0271 - val_loss: 0.0187
Epoch 665/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0268 - val_loss: 0.0174
Epoch 666/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0266 - val_loss: 0.0172
Epoch 667/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0267 - val_loss: 0.0168
Epoch 668/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0268 - val_loss: 0.0180
Epoch 669/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0268 - val_loss: 0.0161
Epoch 670/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0268 - val_loss: 0.0173
Epoch 671/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0262 - val_loss: 0.0199
Epoch 672/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0273 - val_loss: 0.0181
Epoch 673/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0264 - val_loss: 0.0181
Epoch 674/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0264 - val_loss: 0.0164
Epoch 675/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0259 - val_loss: 0.0168
Epoch 676/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0263 - val_loss: 0.0225
Epoch 677/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0273 - val_loss: 0.0169
Epoch 678/700
14818/14818 [==============================] - 45s 3ms/sample - loss: 0.0257 - val_loss: 0.0164
Epoch 679/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0264 - val_loss: 0.0166
Epoch 680/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0256 - val_loss: 0.0160
Epoch 681/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0255 - val_loss: 0.0207
Epoch 682/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0268 - val_loss: 0.0177
Epoch 683/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0254 - val_loss: 0.0183
Epoch 684/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0255 - val_loss: 0.0167
Epoch 685/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0246 - val_loss: 0.0165
Epoch 686/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0248 - val_loss: 0.0163
Epoch 687/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0258 - val_loss: 0.0205
Epoch 688/700
14818/14818 [==============================] - 48s 3ms/sample - loss: 0.0253 - val_loss: 0.0159
Epoch 689/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0254 - val_loss: 0.0193
Epoch 690/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0251 - val_loss: 0.0172
Epoch 691/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0250 - val_loss: 0.0169
Epoch 692/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0255 - val_loss: 0.0192
Epoch 693/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0246 - val_loss: 0.0174
Epoch 694/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0254 - val_loss: 0.0189
Epoch 695/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0248 - val_loss: 0.0151
Epoch 696/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0238 - val_loss: 0.0177
Epoch 697/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0254 - val_loss: 0.0187
Epoch 698/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0256 - val_loss: 0.0162
Epoch 699/700
14818/14818 [==============================] - 47s 3ms/sample - loss: 0.0252 - val_loss: 0.0157
Epoch 700/700
14818/14818 [==============================] - 46s 3ms/sample - loss: 0.0251 - val_loss: 0.0187
********** Successfully loaded weights from weights_695_0.01510.hdf5 file **********
[11]:
model.evaluate_on_training_data(data=data, metrics=["r2", "rmse", "nse"])
***** Training *****
input_x shape: (11854, 15, 7)
target shape: (11854, 1)
11854/11854 [==============================] - 8s 706us/sample
[11]:
{'r2': 0.9897681580986154,
'rmse': 139.5351098290319,
'nse': 0.9896673645075371}
[12]:
model.evaluate_on_test_data(data=data, metrics=["r2", "rmse", "nse"])
***** Test *****
input_x shape: (6352, 15, 7)
target shape: (6352, 1)
6352/6352 [==============================] - 4s 587us/sample
[12]:
{'r2': 0.6147775425572483,
'rmse': 780.0106970420342,
'nse': 0.6143773584544838}
[15]:
test_pred = model.predict_on_training_data(data=data, plots=["regression", "residual", "prediction"])
***** Training *****
input_x shape: (11854, 15, 7)
target shape: (11854, 1)
11854/11854 [==============================] - 8s 674us/sample
[16]:
test_pred = model.predict_on_test_data(data=data, plots=["regression", "residual", "prediction"])
***** Test *****
input_x shape: (6352, 15, 7)
target shape: (6352, 1)
6352/6352 [==============================] - 4s 646us/sample
[7]:
from ai4water.postprocessing import Interpret
[10]:
interpreter = Interpret(model=model)
[11]:
interpreter.interpret_tft(
data=data,
data_type="test",
feature="et_morton_point_SILO",
cbar_params={"pad": 0.5, "border": False}, cmap="jet"
)
***** Test *****
input_x shape: (6352, 15, 7)
target shape: (6352, 1)
***** Test *****
input_x shape: (6352, 15, 7)
target shape: (6352, 1)
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
[11]:
<matplotlib.image.AxesImage at 0x22b39697ac8>
[12]:
ax = interpreter.interpret_tft(
data=data,
data_type="test",
feature="precipitation_AWAP",
cbar_params={"pad": 0.5, "border": False}, cmap="jet"
)
***** Test *****
input_x shape: (6352, 15, 7)
target shape: (6352, 1)
***** Test *****
input_x shape: (6352, 15, 7)
target shape: (6352, 1)
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
[13]:
test_x, test_y = model.test_data(data=data)
***** Test *****
input_x shape: (6352, 15, 7)
target shape: (6352, 1)
[15]:
axes = interpreter.interpret_tft(
x=test_x[0:1000],
y=test_y[0:1000],
feature="et_morton_point_SILO",
cbar_params={"pad": 0.5, "border": False}, cmap="jet"
)
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
[16]:
axes = interpreter.interpret_tft(
x=test_x[0:1000],
y=test_y[0:1000],
feature="precipitation_AWAP",
cbar_params={"pad": 0.5, "border": False}, cmap="jet"
)
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
[17]:
axes = interpreter.interpret_tft(
x=test_x[0:1000],
y=test_y[0:1000],
feature="rh_tmax_SILO",
cbar_params={"pad": 0.5, "border": False}, cmap="jet"
)
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
[18]:
axes = interpreter.interpret_tft(
x=test_x[0:1000],
y=test_y[0:1000],
feature="tmax_AWAP",
cbar_params={"pad": 0.5, "border": False}, cmap="jet"
)
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
[19]:
axes = interpreter.interpret_tft(
x=test_x[0:1000],
y=test_y[0:1000],
feature="tmin_AWAP",
cbar_params={"pad": 0.5, "border": False}, cmap="jet"
)
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
[20]:
axes = interpreter.interpret_tft(
x=test_x[0:1000],
y=test_y[0:1000],
feature="vprp_AWAP",
cbar_params={"pad": 0.5, "border": False}, cmap="jet"
)
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
[21]:
axes = interpreter.interpret_tft(
x=test_x[0:1000],
y=test_y[0:1000],
feature="rh_tmin_SILO",
cbar_params={"pad": 0.5, "border": False}, cmap="jet"
)
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
[26]:
attention_weights, _ = interpreter.tft_attention_components(
data=data,
data_type="test")
***** Test *****
input_x shape: (6352, 15, 7)
target shape: (6352, 1)
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.
[27]:
for k,v in attention_weights.items():
print(k, v.shape)
decoder_self_attn (6, 6352, 15, 15)
encoder_variable_selection_weights (6352, 15, 7)
[33]:
enc_weights = attention_weights['encoder_variable_selection_weights']
[40]:
enc_weights_avg_lb = np.mean(enc_weights, axis=1)
enc_weights_avg_lb.shape
[40]:
(6352, 7)
[39]:
_ = imshow(enc_weights_avg_lb.transpose(),
colorbar=True, cmap="jet",
cbar_params={"border": False},
aspect="auto",
yticklabels=model.input_features,
ax_kws=dict(xlabel="Examples"),
)
[ ]: