Elmo Embedding model for TensorFlow 2

General Discussion
, ,
#1

I am trying to use Elmo Embedding model Using TensorFlow 2

# Imported Elmo Layer
elmo_model_path = "https://tfhub.dev/google/elmo/3"
elmo_layer = hub.KerasLayer(elmo_model_path, input_shape=[], dtype=tf.string, trainable=False)

model = tf.keras.Sequential([
    elmo_layer,
    tf.keras.layers.Dense(8, activation='sigmoid'),
    tf.keras.layers.Dense(1, activation='sigmoid')
])

# Setting hyperparameter  
optimizer = tf.keras.optimizers.Adam(learning_rate=0.01)
model.compile(loss='binary_crossentropy',optimizer=optimizer,metrics=['accuracy'])

# Model Summary
model.summary()

#Data
data = ['our deeds reason earthquake may allah forgive us', 'forest fire near la ronge sask canada', 'all residents asked shelter place notified officers no evacuation shelter place orders expected', ' people receive wildfires evacuation orders california', 'just got sent photo ruby alaska smoke wildfires pours school', 'rockyfire update  california hwy  closed directions due lake county fire  cafire wildfires', 'flood disaster heavy rain causes flash flooding streets manitou colorado springs areas', 'im top hill i can see fire woods', 'theres emergency evacuation happening now building across street', 'im afraid tornado coming area', 'three people died heat wave far']
['our deeds reason earthquake may allah forgive us', 'forest fire near la ronge sask canada', 'all residents asked shelter place notified officers no evacuation shelter place orders expected', ' people receive wildfires evacuation orders california', 'just got sent photo ruby alaska smoke wildfires pours school']
label = ['1', '1', '1', '1', '1']

#converting the labels to int value
label = list(map(np.int64, label))

#Creating Training Dataset
training_data = tf.data.Dataset.from_tensor_slices((data,label)).prefetch(1)

print(type(training_data))
print(training_data)

# Training
num_epochs = 5
history = model.fit(training_data.shuffle(10000).batch(2), epochs=num_epochs, verbose=2)

Error

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_24964/3287467954.py in <module>
      1 num_epochs = 5
----> 2 history = model.fit(training_data.shuffle(10000).batch(2), epochs=num_epochs, verbose=2)

~\anaconda3\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py in shuffle(self, buffer_size, seed, reshuffle_each_iteration, name)
   1488       Dataset: A `Dataset`.
   1489     """
-> 1490     return ShuffleDataset(
   1491         self, buffer_size, seed, reshuffle_each_iteration, name=name)
   1492 

~\anaconda3\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py in __init__(self, input_dataset, buffer_size, seed, reshuffle_each_iteration, name)
   4802           **self._common_args)
   4803     else:
-> 4804       variant_tensor = gen_dataset_ops.shuffle_dataset(
   4805           input_dataset._variant_tensor,  # pylint: disable=protected-access
   4806           buffer_size=self._buffer_size,

~\anaconda3\lib\site-packages\tensorflow\python\ops\gen_dataset_ops.py in shuffle_dataset(input_dataset, buffer_size, seed, seed2, output_types, output_shapes, reshuffle_each_iteration, metadata, name)
   6835     metadata = ""
   6836   metadata = _execute.make_str(metadata, "metadata")
-> 6837   _, _, _op, _outputs = _op_def_library._apply_op_helper(
   6838         "ShuffleDataset", input_dataset=input_dataset,
   6839                           buffer_size=buffer_size, seed=seed, seed2=seed2,

~\anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py in _apply_op_helper(op_type_name, name, **keywords)
    506                   preferred_dtype=default_dtype)
    507           else:
--> 508             values = ops.convert_to_tensor(
    509                 values,
    510                 name=input_arg.name,

~\anaconda3\lib\site-packages\tensorflow\python\profiler\trace.py in wrapped(*args, **kwargs)
    181         with Trace(trace_name, **trace_kwargs):
    182           return func(*args, **kwargs)
--> 183       return func(*args, **kwargs)
    184 
    185     return wrapped

~\anaconda3\lib\site-packages\tensorflow\python\framework\ops.py in convert_to_tensor(value, dtype, name, as_ref, preferred_dtype, dtype_hint, ctx, accepted_result_types)
   1652       graph = get_default_graph()
   1653       if not graph.building_function:
-> 1654         raise RuntimeError("Attempting to capture an EagerTensor without "
   1655                            "building a function.")
   1656       return graph.capture(value, name=name)

RuntimeError: Attempting to capture an EagerTensor without building a function.
#3

Unfortunately at this time the authors of Elmo are not planning to release TF2 version of the model.

Please refer TensorFlow Hub documentation to check the text embeddings supported in version 1 and version 2. Thank you

1 Like
#5

Below code should work.

============================================
import tensorflow as tf
import tensorflow_hub as hub
import numpy as np

elmo = hub.load(“Find Pre-trained Models | Kaggle”]

x = [“Hello world!”]
embeddings = elmo(tf.constant(x))[“elmo”]

print(embeddings.shape)
print(embeddings.numpy())

============================================