Using convolutional neural networks part 2

Training your model

In the previous post we exported the data to the Google Cloud Storage. In this post will read the data and use the data to run a model. There are many examples using colab. Here we focus on training your model on your pc or a server.

download the data from the gcp bucket.

Download the data from the gcp bucket.

use gsutil to copy the files to your folder of choice on your pc

gsutil -m  cp -r gs://wfpbucket/l8Records myoutput/data/folder

install the tensorflow python package. Note that you can install tensorflow-gpu if your system has a powerfull GPU. You will also need to install the NVIDIA drivers if you would like to take advantage of the GPU.

pip install tensorflow

copy and paste the model below and set the correct paths.

# Tensorflow setup.
import tensorflow as tf

from tensorflow.python.keras import layers
from tensorflow.python.keras import losses
from tensorflow.python.keras import models
from tensorflow.python.keras import metrics
from tensorflow.python.keras import optimizers

def parse_tfrecord(example_proto):
    """The parsing function.
    Read a serialized example into the structure defined by FEATURES_DICT.
    example_proto: a serialized Example.
    A dictionary of tensors, keyed by feature name.
    return, FEATURES_DICT)

def to_tuple(inputs):
    """Function to convert a dictionary of tensors to a tuple of (inputs, outputs).
    Turn the tensors returned by parse_tfrecord into a stack in HWC shape.
    inputs: A dictionary of tensors, keyed by feature name.
    A tuple of (inputs, outputs).
    inputsList = [inputs.get(key) for key in FEATURES]
    stacked = tf.stack(inputsList, axis=0)
    # Convert from CHW to HWC
    stacked = tf.transpose(stacked, [1, 2, 0])
    return stacked[:,:,:len(BANDS)], stacked[:,:,len(BANDS):]

def get_dataset(pattern):
    """Function to read, parse and format to tuple a set of input tfrecord files.
    Get all the files matching the pattern, parse and convert to tuple.
    pattern: A file pattern to match in a Cloud Storage bucket.
    glob =
    dataset =, compression_type='GZIP')
    dataset =, num_parallel_calls=5)
    dataset =, num_parallel_calls=5)
    return dataset

def get_training_dataset(glob,eval=True):
    """Get the preprocessed training dataset
    A of training data.
   dataset = get_dataset(glob)
   if eval:
       dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE).repeat()
        dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE)

    return dataset

def conv_block(input_tensor, num_filters):
    encoder = layers.Conv2D(num_filters, (3, 3), padding='same')(input_tensor)
    encoder = layers.BatchNormalization()(encoder)
    encoder = layers.Activation('relu')(encoder)
    encoder = layers.Conv2D(num_filters, (3, 3), padding='same')(encoder)
    encoder = layers.BatchNormalization()(encoder)
    encoder = layers.Activation('relu')(encoder)
    return encoder

def encoder_block(input_tensor, num_filters):
    encoder = conv_block(input_tensor, num_filters)
    encoder_pool = layers.MaxPooling2D((2, 2), strides=(2, 2))(encoder)
    return encoder_pool, encoder

def decoder_block(input_tensor, concat_tensor, num_filters):
    decoder = layers.Conv2DTranspose(num_filters, (2, 2), strides=(2, 2), padding='same')(input_tensor)
    decoder = layers.concatenate([concat_tensor, decoder], axis=-1)
    decoder = layers.BatchNormalization()(decoder)
    decoder = layers.Activation('relu')(decoder)
    decoder = layers.Conv2D(num_filters, (3, 3), padding='same')(decoder)
    decoder = layers.BatchNormalization()(decoder)
    decoder = layers.Activation('relu')(decoder)
    decoder = layers.Conv2D(num_filters, (3, 3), padding='same')(decoder)
    decoder = layers.BatchNormalization()(decoder)
    decoder = layers.Activation('relu')(decoder)
    return decoder

def get_model():
    n = 4
    inputs = layers.Input(shape=[None, None, len(BANDS)]) 
    encoder0_pool, encoder0 = encoder_block(inputs, n) 
    encoder1_pool, encoder1 = encoder_block(encoder0_pool, n*2) 
    encoder2_pool, encoder2 = encoder_block(encoder1_pool, n*4)
    encoder3_pool, encoder3 = encoder_block(encoder2_pool, n*8)
    encoder4_pool, encoder4 = encoder_block(encoder3_pool, n*16)
    center = conv_block(encoder4_pool, n*32) # center
    decoder4 = decoder_block(center, encoder4, n*16) 
    decoder3 = decoder_block(decoder4, encoder3, n*8) 
    decoder2 = decoder_block(decoder3, encoder2, n*4)
    decoder1 = decoder_block(decoder2, encoder1, n*2)
    decoder0 = decoder_block(decoder1, encoder0, n)
    outputs = layers.Conv2D(5, (1, 1), activation='softmax')(decoder0)

    model = models.Model(inputs=[inputs], outputs=[outputs])


        optimizer = 'ADAM', 
        loss = losses.categorical_crossentropy,
        metrics = [metrics.categorical_accuracy])

    return model
# Specify inputs (Landsat bands) to the model and the response variable.
opticalBands = ['B2', 'B3', 'B4', 'B5', 'B6', 'B7']

BANDS = opticalBands
RESPONSE = ['cloud','shadow','snow','water','land']
# Specify the size and shape of patches expected by the model.
COLUMNS = [, dtype=tf.float32) for k in FEATURES

# Specify model training parameters.
TRAIN_SIZE = 6000 # train size is 80*100*0.7
BUFFER_SIZE = 4000  
# location of the gcp bucket
data_path = "/path/to/data/"

# import the training, testing and validation records
training_files = data_path + '/training/train*'
testing_files = data_path + '/testing/test*'
validation_files = data_path + '/validation/val*'  

training_ds = get_training_dataset(training_files)
testing_ds = get_training_dataset(testing_files)  
validation_ds = get_training_dataset(validation_files,False)

model = get_model()
   x = training_ds, 
   epochs = EPOCHS, 
   steps_per_epoch =int(TRAIN_SIZE / BATCH_SIZE), 
   validation_data = testing_ds,
   validation_steps = 100)


# save model
MODEL_DIR = '/path/to/save/qamodel/', MODEL_DIR)

After the model finished we need to make it readable for the earth engine

from import saved_model_utils

MODEL_DIR = r'/path/to/qamodel'
label = "landclass"

meta_graph_def = saved_model_utils.get_meta_graph_def(MODEL_DIR, 'serve')
inputs = meta_graph_def.signature_def['serving_default'].inputs
outputs = meta_graph_def.signature_def['serving_default'].outputs

# Just get the first thing(s) from the serving signature def.  i.e. this
# model only has a single input and a single output.
input_name = None
for k,v in inputs.items():
    input_name =

output_name = None
for k,v in outputs.items():
    output_name =

# Make a dictionary that maps Earth Engine outputs and inputs to 
# AI Platform inputs and outputs, respectively.
import json
input_dict = "'" + json.dumps({input_name: "array"}) + "'"
output_dict = "'" + json.dumps({output_name: label}) + "'"


We use the information from the code above to run the lines below.

earthengine model prepare --source_dir /paht/to/qamodel/ --dest /path/to/qaEefied --input "{\"serving_default_input_1:0\":\"array\"}" --output "{\"StatefulPartitionedCall:0\":\"landclass\"}"

Now we upload the model to the Google Cloud Platform and run the code below to push the model to the ai platform. You may need to create your project and model on the cloud platform.

gcloud ai-platform versions create mymodelversion --project projectName --model modelName --origin gs://mybucket/qaEefied --runtime-version=2.3 --framework "TENSORFLOW" --python-version=3.7

One comment

Leave a Reply