How do I load pre-trained embeddings?

[samuelhkahn]

I was wondering what is the proper way to load pre-trained embeddings using Keras-Tensorflow with SageMaker. Normally you would load pretrained embeddings (such as GLove) into memory and then assign them to your embedding layer as follows:

embedding = layers.Embedding(50000,300), weights=[embedding_matrix])(text)

where embedding_matrix is a (50k,300) pretrained embedding matrix. But i'm not sure how to actually load the embedding matrix into memory in the keras_model_fn function in the entry point file. Help would be appreciated it. My entrypoint file is as follows:

import numpy as np
import os
import json
import pickle
import sys
import traceback
import tensorflow as tf
from tensorflow.python.estimator.export.export import build_raw_serving_input_receiver_fn
from tensorflow.python.keras._impl.keras.layers import Dense
from tensorflow.python.keras._impl.keras.layers import Dropout
from tensorflow.python.keras._impl.keras.layers import LSTM
from tensorflow.python.keras._impl.keras.layers.embeddings import Embedding
from tensorflow.python.keras._impl.keras.optimizers import Adam
from tensorflow.python.keras._impl.keras.callbacks import ModelCheckpoint
from tensorflow.python.keras._impl.keras.callbacks import CSVLogger
from tensorflow.python.keras._impl.keras.callbacks import EarlyStopping
from tensorflow.python.keras._impl.keras.callbacks import LambdaCallback
from tensorflow.python.keras._impl.keras import metrics
from tensorflow.python.keras._impl.keras.models import Model
from tensorflow.python.keras._impl.keras import layers
from tensorflow.python.keras._impl.keras import Input

NUM_CLASSES = 2
NUM_DATA_BATCHES = 5
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 10000 * NUM_DATA_BATCHES
BATCH_SIZE = 256
INPUT_TENSOR_NAME_1 = 'text1' # needs to match the name of the first layer + "_input"
INPUT_TENSOR_NAME_2 = 'text2' # needs to match the name of the first layer + "_input"
INPUT_TENSOR_NAME_3 = 'title1' # needs to match the name of the first layer + "_input"
INPUT_TENSOR_NAME_4 = 'title2' # needs to match the name of the first layer + "_input"



def keras_model_fn(training_dir):
    """keras_model_fn receives hyperparameters from the training job and returns a compiled keras model.
    The model will transformed in a TensorFlow Estimator before training and it will saved in a TensorFlow Serving
    SavedModel in the end of training.

    Args:
        hyperparameters: The hyperparameters passed to SageMaker TrainingJob that runs your TensorFlow training
                         script.
    Returns: A compiled Keras model
    """

    text_input_1 = Input(shape=(None,), dtype='int32', name='text1')
    embedded_text_1 = layers.Embedding(50000,300)(text_input_1)
    embed_drop_1=Dropout(.5)(embedded_text_1)

    text_input_2 = Input(shape=(None,), dtype='int32', name='text2')
    embedded_text_2 = layers.Embedding(50000,300,)(text_input_2)
    embed_drop_2=Dropout(.5)(embedded_text_2)


    shared_lstm_text = LSTM(256)
    left_output_text = shared_lstm_text(embed_drop_1)
    right_output_text = shared_lstm_text(embed_drop_2)

    title_input_1 = Input(shape=(None,), dtype='int32', name='title1')
    embedded_title_1 = layers.Embedding(50000,300)(title_input_1)
    embed_drop_3=Dropout(.5)(embedded_title_1)

    title_input_2 = Input(shape=(None,), dtype='int32', name='title2')
    embedded_title_2 = layers.Embedding(50000,300)(title_input_2)
    embed_drop_4=Dropout(.5)(embedded_title_2)

    shared_lstm_title = LSTM(128)
    left_output_title = shared_lstm_title(embed_drop_3)
    right_output_title = shared_lstm_title(embed_drop_4)
    # Calculates the distance as defined by the MaLSTM model
    # malstm_distance = Merge(mode=lambda x: exponent_neg_manhattan_distance(x[0], x[1]), output_shape=lambda x: (x[0][0], 1))([left_output, right_output])
    merged = layers.concatenate([left_output_text, right_output_text,left_output_title, right_output_title], axis=-1)
    drop_1 = Dropout(.3)(merged)
    dense_1 = layers.Dense(256, activation='sigmoid')(drop_1)
    drop_2 = Dropout(.3)(dense_1)

    dense_2 = layers.Dense(128, activation='sigmoid')(drop_2)


    predictions = layers.Dense(1, activation='sigmoid')(dense_2)

    # Pack it all up into a model
    shared_layer_model = Model([text_input_1, text_input_2,title_input_1,title_input_2], [predictions])
    shared_layer_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
    return shared_layer_model


def train_input_fn(training_dir , hyperparameters = None):

    return _input_fn(training_dir,"train")

def eval_input_fn(training_dir , hyperparameters = None):

    return _input_fn(training_dir,"dev")

def serving_input_fn(hyperparameters = None):

    text_ph_1 = tf.placeholder(tf.int32, shape=[None,500])
    text_ph_2 = tf.placeholder(tf.int32, shape=[None,500])
    title_ph_1 = tf.placeholder(tf.int32, shape=[None,20])
    title_ph_2 = tf.placeholder(tf.int32, shape=[None,20])

    #label is not required since serving is only used for inference
    feature_placeholders = {"text1":text_ph_1,"text2":text_ph_2,"title1":title_ph_1,"title2":title_ph_2}
    return build_raw_serving_input_receiver_fn(feature_placeholders)()

def _input_fn(training_dir,mode):


    if mode=="train":
        train_text_1=np.vstack((np.load(training_dir+"/negative_"+mode+"_text_1.npy"),np.load(training_dir+"/positive_"+mode+"_text_1.npy")))
        train_text_2=np.vstack((np.load(training_dir+"/negative_"+mode+"_text_2.npy"),np.load(training_dir+"/positive_"+mode+"_text_2.npy")))
    else:
        train_text_1=np.load(training_dir+"/"+mode+"_text_1.npy")
        train_text_2=np.load(training_dir+"/"+mode+"_text_2.npy")
    train_title_1=np.load(training_dir+"/"+mode+"_title_1.npy")
    train_title_2=np.load(training_dir+"/"+mode+"_title_2.npy")

    y=np.load(training_dir+"/"+mode+"_targets.npy")
    y=y.reshape((y.shape[0],1)).astype(np.float32)

    permutation = np.random.permutation(train_text_1.shape[0])

    train_text_1=train_text_1[permutation]
    train_text_2=train_text_2[permutation]

    train_title_1=train_title_1[permutation]
    train_title_2=train_title_2[permutation]

    y=y[permutation]

    x={INPUT_TENSOR_NAME_1: train_text_1, 
       INPUT_TENSOR_NAME_2: train_text_2,
       INPUT_TENSOR_NAME_3: train_title_1, 
       INPUT_TENSOR_NAME_4: train_title_2}
    dataset=tf.estimator.inputs.numpy_input_fn(x=x,y=y,batch_size=BATCH_SIZE,num_epochs=10,shuffle=False)()


    return dataset

[zmjjmz]

Hey there,

Not anyone official but I've been doing this through a channel where I serialize my embeddings and then load them back up in the model_fn.

Additionally, you shouldn't use the (apparently deprecated) weights argument, as it does funky stuff when recreating the graph for serving. I've instead been setting embeddings_initializer to a constant value as so:

emeddings_initializer=keras.initializers.Constant(value=embedding_matrix).

[samuelhkahn]

@zmjjmz Thanks for the response man! I guess what i am confused about is how you actually load them back up in model_fn... I can't seem to get my serialized weights (.npy) to load. What does your loading line look like?

Thanks for the tip, I will definitely updated to the non-deprecated version!

[zmjjmz]

I just have them serialized as a pickle. How are you calling the fit method from the SDK? If you're using channels (a dict of str:S3 URI) then for example you might have it look like

{
 'training':'s3://your-bucket/training/'
 'embeddings':'s3://your-bucket/embeddings/'

And then put your .npy into the latter (and your dataset into the former). Basically the container then downloads these S3 folders locally and runs off of that, so you can reference it according to a base path (in the container it should be /opt/ml/input/data).

The former is by default parsed into the training_dir that you get, and then the latter you could get by

os.path.join(BASE_PATH, 'embeddings', 'embeddings.npy')

Deserializing that path should work.

[samuelhkahn]

@zmjjmz thanks again for the response, sorry for the delay got caught up with a couple things.

I am calling the fit method first by creating

    estimator = TensorFlow(
        entry_point='entry_point_file.py',
        role="my_role",
        training_steps=1000,
        evaluation_steps=100,
        hyperparameters={'throttle_secs': 30},
        train_instance_count=1,
        train_instance_type='ml.p2.8xlarge',
        train_volume_size=200,
        base_job_name='siamese_lstm')

    estimator.fit(TRAING_DATA_BUCKET,run_tensorboard_locally=True)

looking at the fit method:

        """Train a model using the input training dataset.
        The API calls the Amazon SageMaker CreateTrainingJob API to start model training.
        The API uses configuration you provided to create the estimator and the
        specified input training data to send the CreatingTrainingJob request to Amazon SageMaker.
        This is a synchronous operation. After the model training successfully completes,
        you can call the ``deploy()`` method to host the model using the Amazon SageMaker hosting services.
        Args:
            inputs (str or dict or sagemaker.session.s3_input): Information about the training data.
                This can be one of three types:
                * (str) the S3 location where training data is saved.
                * (dict[str, str] or dict[str, sagemaker.session.s3_input]) If using multiple channels for
                    training data, you can specify a dict mapping channel names
                    to strings or :func:`~sagemaker.session.s3_input` objects.
                * (sagemaker.session.s3_input) - channel configuration for S3 data sources that can provide
                    additional information about the training dataset. See :func:`sagemaker.session.s3_input`
                    for full details.
            wait (bool): Whether the call should wait until the job completes (default: True).
            logs (bool): Whether to show the logs produced by the job.
                Only meaningful when wait is True (default: True).
            job_name (str): Training job name. If not specified, the estimator generates a default job name,
                based on the training image name and current timestamp.
        """

it looks like I might need to specify the embedding channel in the inputs argument.

[zmjjmz]

Yes, that's where the dictionary I mentioned earlier would go.

[samuelhkahn]

Thanks for the help @zmjjmz

[ChoiByungWook]

Thank you @zmjjmz for providing assistance!