# Model

## Keras

#### **get the output of middle layer**

Since we know keras model needs to be built and compiled before executing compared to eager execution in TF2 or dynamic graph in PyTorch, therefore, we need to build a new model to get the output of the middle layer. It is easier than PyTorch since we can create a sub graph using the original keras model and will not modify the original model.

```python
> [layer.name for layer in tf_model.layers]
# ['Input-Token',
#  'Input-Segment',
#  'Embedding-Token',
#  'Embedding-Segment',
#  .......
#  'Transformer-11-FeedForward-Dropout',
#  'Transformer-11-FeedForward-Add',
#  'Transformer-11-FeedForward-Norm',
#  **************************************************
#  ** in case we want the output before last layer **
#  **************************************************
#  'dense']

> layer_model = keras.models.Model(inputs=tf_model.input, outputs=tf_model.get_layer('Transformer-11-FeedForward-Norm').output)
> layer_output = layer_model(x_input)
```

#### **print the attributes of Model**

```python
> type(keras_model)
# keras.engine.functional.Functional


> keras_model.tf_model.get_config()
# {'name': 'model_1',
# 'layers': [{'class_name': 'InputLayer',
# 'config': {'batch_input_shape': (None, None),
#     'dtype': 'float32',
#     'sparse': False,
#     'ragged': False,
#     'name': 'Input-Token'},
# 'name': 'Input-Token',
# 'inbound_nodes': []},
# {'class_name': 'InputLayer',
# 'config': {'batch_input_shape': (None, None),
#     'dtype': 'float32',
#     'sparse': False,
#     'ragged': False,
#     'name': 'Input-Segment'},
# 'name': 'Input-Segment',
# 'inbound_nodes': []},
# ...
# {'class_name': 'Dense',
# 'config': {'name': 'dense',
#  'trainable': True,
#  'dtype': 'float32',
#  'units': 22,
#  'activation': 'softmax',
#  'use_bias': True,
#  'kernel_initializer': {'class_name': 'GlorotUniform',
#   'config': {'seed': None}},
#  'bias_initializer': {'class_name': 'Zeros', 'config': {}},
#  'kernel_regularizer': None,
#  'bias_regularizer': None,
#  'activity_regularizer': None,
#  'kernel_constraint': None,
#  'bias_constraint': None},
# 'name': 'dense',
# 'inbound_nodes': [[['Transformer-11-FeedForward-Norm', 0, 0, {}]]]}],
#  'input_layers': [['Input-Token', 0, 0], ['Input-Segment', 0, 0]],
#  'output_layers': [['dense', 0, 0]]}


> tf_model.summary()
# Model: "model_1"
# __________________________________________________________________________________________________
# Layer (type)                   Output Shape         Param #     Connected to                     
# ==================================================================================================
# Input-Token (InputLayer)       [(None, None)]       0           []                               
# Input-Segment (InputLayer)     [(None, None)]       0           []                               
# Embedding-Token (Embedding)    (None, None, 768)    384885504   ['Input-Token[0][0]']            
# Embedding-Segment (Embedding)  (None, None, 768)    1536        ['Input-Segment[0][0]']     
# ................
# dense (Dense)                  (None, None, 22)     16918       ['Transformer-11-FeedForward-Norm[0][0]']      
# ==================================================================================================
# Total params: 470,353,174
# Trainable params: 470,353,174
# Non-trainable params: 0
# __________________________________________________________________________________________________     


> tf_model.layers
# [<keras.engine.input_layer.InputLayer at 0x7ff158495370>,
#  <keras.engine.input_layer.InputLayer at 0x7fef2a285fd0>,
#  <bert4keras.layers.Embedding at 0x7fef2a2b35e0>,
#  <bert4keras.layers.Embedding at 0x7fef2a2b3df0>,
#  <keras.layers.merge.Add at 0x7fef2a2b3820>,
#  <bert4keras.layers.PositionEmbedding at 0x7fef289d2be0>,
#  <bert4keras.layers.LayerNormalization at 0x7fe
#  ................
#  <keras.layers.core.dense.Dense at 0x7ff158495520>]


> [layer.name for layer in tf_model.layers]
# ['Input-Token',
#  'Input-Segment',
#  'Embedding-Token',
#  'Embedding-Segment',
#  'Embedding-Token-Segment',
#  'Embedding-Position',
#  'Embedding-Norm',
#  ................
#  'dense']


> type(tf_model.get_layer('dense'))
# keras.layers.core.dense.Dense


> tf_model.get_layer('dense').weights
# [<tf.Variable 'dense/kernel:0' shape=(768, 22) dtype=float32, numpy=
#  array([[-0.00010732, -0.0852981 , -0.04779567, ..., -0.04220317,
#          ...,
#          [-0.05151889, -0.01136582,  0.03139671, ...,  0.00703724,
#  <tf.Variable 'dense/bias:0' shape=(22,) dtype=float32, numpy=
#  array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
#          0., 0., 0., 0., 0.], dtype=float32)>]


> tf_model.get_layer('dense').variables
# (Return the same as weight in this situation, but they have the different id(method))
```