Data Science Asked by Uday on February 12, 2021
I am writing Encoder-Decoder architecture with Bahdanau Attention using tf.keras with TensorFlow 2.0. Below is my code This is working with TensorFlow 1.15 but getting the error in 2.0. you can check the code in colab notebook here.
can you please tell me what is wrong in the code.
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow.keras.layers import Input, Dense, Conv2D, BatchNormalization, Activation, Dropout, GRU, Embedding
from tensorflow.keras.models import Model
from tensorflow.keras import activations
from tensorflow.keras.layers import Layer
from tensorflow.keras import layers
import tensorflow as tf
from tensorflow.keras.layers import GRU, concatenate, Lambda
ENCODER_SEQ_LEN = 30
DECODER_SEQ_LEN = 20
VOCAB_SIZE = 500
units = 16
tf.keras.backend.clear_session()
class Encoder(Model):
def __init__(self, vocab_size, embedding_dim, input_length, units):
super(Encoder, self).__init__()
self.vocab_size = vocab_size
self.embedding_dim = embedding_dim
self.input_length = input_length
self.units = units
self.embedding = Embedding(input_dim=VOCAB_SIZE, output_dim=50, input_length=self.input_length,
mask_zero=False, name="embedding_layer_encoder")
self.gru = GRU(self.units, return_state=True, return_sequences=True, name="Encoder_GRU")
@tf.function
def call(self, inputs, training=True):
x_embedd = self.embedding(inputs)
gru_output, gru_state = self.gru(x_embedd)
return gru_output, gru_state
class BahdanauAttention(tf.keras.layers.Layer):
def __init__(self, units):
super(BahdanauAttention, self).__init__()
self.W1 = tf.keras.layers.Dense(units)
self.W2 = tf.keras.layers.Dense(units)
self.V = tf.keras.layers.Dense(1)
def call(self, query, values):
# hidden shape == (batch_size, hidden size)
# # hidden_with_time_axis shape == (batch_size, 1, hidden size)
# # we are doing this to perform addition to calculate the score
hidden_with_time_axis = tf.expand_dims(query, 1)
# score shape == (batch_size, max_length, 1)
# # we get 1 at the last axis because we are applying score to self.V
# # the shape of the tensor before applying self.V is (batch_size, max_length, units)
score = self.V(tf.nn.tanh(self.W1(values) + self.W2(hidden_with_time_axis)))
# attention_weights shape == (batch_size, max_length, 1)
attention_weights = tf.nn.softmax(score, axis=1)
# context_vector shape after sum == (batch_size, hidden_size)
context_vector = attention_weights * values
context_vector = tf.reduce_sum(context_vector, axis=1)
return context_vector
class onestepDecoder(Model):
def __init__(self, vocab_size, embedding_dim, dec_units, att_units):
super(onestepDecoder, self).__init__()
self.vocab_size = vocab_size
self.embedding_dim = embedding_dim
self.dec_units = dec_units
self.att_units = att_units
self.embedd = Embedding(input_dim=self.vocab_size, output_dim=self.embedding_dim,
input_length=1, mask_zero=False, name="Decoder_Embedding_layer")
self.att_layer = BahdanauAttention(units=self.att_units) #name='Attention')
self.dense = Dense(self.vocab_size, activation="softmax", name="DenseOut")
self.gru = GRU(units=self.dec_units, return_state=True, name="DecGRU")
@tf.function
def call(self, input_decoder, input_state, encoder_outputs, training=True):
x_embedd = self.embedd(input_decoder)
context_vector = self.att_layer(input_state, encoder_outputs )
concat = tf.concat([tf.expand_dims(context_vector, 1), x_embedd], axis=-1)
decoder_output, Decoder_state = self.gru(concat, initial_state=input_state)
output = self.dense(decoder_output)
return (output, Decoder_state)
class Decoder(Model):
def __init__(self, vocab_size, embedding_dim, dec_units, att_units):
super(Decoder, self).__init__()
self.vocab_size = vocab_size
self.embedding_dim = embedding_dim
self.dec_units = dec_units
self.att_units = att_units
self.stepdec = onestepDecoder(self.vocab_size, self.embedding_dim, self.dec_units, self.att_units)
@tf.function
def call(self, input_decoder, input_state, encoder_outputs):
all_outputs= tf.TensorArray(tf.float32, size=input_decoder.shape[1], name="output_arrays")
for timestep in range(input_decoder.shape[1]):
output, input_state = self.stepdec(input_decoder[:,timestep:timestep+1], input_state, encoder_outputs)
all_outputs = all_outputs.write(timestep, output)
all_outputs = tf.transpose(all_outputs.stack(), [1, 0, 2])
return all_outputs
encoder_input = Input(shape=(ENCODER_SEQ_LEN,), name='encoder_input_final')
decoder_input = Input(shape=(DECODER_SEQ_LEN,), name="Decoder_inout_final")
encoder = Encoder(vocab_size=VOCAB_SIZE, embedding_dim=50, input_length=ENCODER_SEQ_LEN, units=16)
x_gru_out, x_gru_state = encoder(encoder_input)
decoder = Decoder(vocab_size=VOCAB_SIZE, embedding_dim=50, dec_units=16, att_units=20)
all_outputs = decoder(decoder_input, x_gru_state, x_gru_out)
encoder_decoder = Model([encoder_input, decoder_input], outputs=all_outputs)
encoder_decoder.compile(optimizer='adam',loss='sparse_categorical_crossentropy')
x = np.random.randint(0, 499, size=(2000, ENCODER_SEQ_LEN))
y = np.random.randint(0, 499, size=(2000, DECODER_SEQ_LEN))
encoder_decoder.fit(x=[x,y], y=y, epochs=1,verbose=1,batch_size=32)
Error:
TypeError Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
60 op_name, inputs, attrs,
—> 61 num_outputs)
62 except core._NotOkStatusException as e:
TypeError: An op outside of the function building code is being passed
a “Graph” tensor. It is possible to have Graph tensors
leak out of the function building context by including a
tf.init_scope in your function building code.
For example, the following function will fail:
@tf.function
def has_init_scope():
my_constant = tf.constant(1.)
with tf.init_scope():
added = my_constant * 2
The graph tensor has name: keras_learning_phase:0
During handling of the above exception, another exception occurred:
_SymbolicException Traceback (most recent call last)
11 frames
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
73 raise core._SymbolicException(
74 “Inputs to eager execution function cannot be Keras symbolic ”
—> 75 “tensors, but found {}”.format(keras_symbolic_tensors))
76 raise e
77 # pylint: enable=protected-access
_SymbolicException: Inputs to eager execution function cannot be Keras symbolic tensors, but found []
It is, as you noticed, about the new version that changed the backend, now using something called eager execution.
The discussion on GitHub has some solutions though. Some suggested it is about incompatibility between the layers. But as the title of the issue implies it is a pain in the neck.
What may solve the problem is disabling it, which works in TF 2.1
, but not in 2.3
.
model.compile(..., experimental_run_tf_function=False)
The following may work in newer versions:
tf.config.experimental_run_functions_eagerly(True)
or you can try this one:
import tensorflow as tf
tf.compat.v1.disable_eager_execution()
None worked for me, so I changed the backend to theano
.
Answered by fuadcan on February 12, 2021
Get help from others!
Recent Questions
Recent Answers
© 2024 TransWikia.com. All rights reserved. Sites we Love: PCI Database, UKBizDB, Menu Kuliner, Sharing RPP