'LayersConfig',

'TF_GRAPHKEYS_VARIABLES',

'flatten_reshape',

'clear_layers_name',

'set_name_reuse',

'initialize_rnn_state',

'print_all_variables',

'get_variables_with_name',

'get_layers_with_name',

'list_remove_repeat',

'merge_networks',

'initialize_global_variables',

'Layer',

'InputLayer',

'OneHotInputLayer',

'Word2vecEmbeddingInputlayer',

'EmbeddingInputlayer',

'AverageEmbeddingInputlayer',

'DenseLayer',

'ReconLayer',

'DropoutLayer',

'GaussianNoiseLayer',

'DropconnectDenseLayer',

tf_dtype = tf.float32 # TensorFlow DType

set_keep = {} # A dictionary for holding tf.placeholders

@abstractmethod

def __init__(self):

"""Reshapes a high-dimension vector input.

dim = 1

for d in variable.get_shape()[1:].as_list():

dim *= d

"""Returns the initialized RNN state.

try: # TF1.0

LSTMStateTuple = tf.contrib.rnn.LSTMStateTuple

except Exception:

LSTMStateTuple = tf.nn.rnn_cell.LSTMStateTuple

if isinstance(state, LSTMStateTuple):

c = state.c.eval(feed_dict=feed_dict)

h = state.h.eval(feed_dict=feed_dict)

return (c, h)

else:

new_state = state.eval(feed_dict=feed_dict)

"""Print information of trainable or all variables,

# tvar = tf.trainable_variables() if train_only else tf.all_variables()

if train_only:

t_vars = tf.trainable_variables()

logging.info(" [*] printing trainable variables")

else:

try: # TF1.0+

t_vars = tf.global_variables()

except Exception: # TF0.12

t_vars = tf.all_variables()

logging.info(" [*] printing global variables")

for idx, v in enumerate(t_vars):

"""Get a list of TensorFlow variables by a given name scope.

if name is None:

raise Exception("please input a name")

logging.info(" [*] geting variables with %s" % name)

# tvar = tf.trainable_variables() if train_only else tf.all_variables()

if train_only:

t_vars = tf.trainable_variables()

else:

try: # TF1.0+

t_vars = tf.global_variables()

except Exception: # TF0.12

t_vars = tf.all_variables()

d_vars = [var for var in t_vars if name in var.name]

if printable:

for idx, v in enumerate(d_vars):

logging.info(" got {:3}: {:15} {}".format(idx, v.name, str(v.get_shape())))

"""Get a list of layers' output in a network by a given name scope.

logging.info(" [*] geting layers with %s" % name)

layers = []

i = 0

for layer in net.all_layers:

# logging.info(type(layer.name))

if name in layer.name:

layers.append(layer)

if printable:

logging.info(" got {:3}: {:15} {}".format(i, layer.name, str(layer.get_shape())))

i = i + 1

"""Remove the repeated items in a list, and return the processed list.

y = []

for i in x:

if not i in y:

y.append(i)

"""Merge all parameters, layers and dropout probabilities to a :class:`Layer`.

if layers is None:

raise Exception("layers should be a list of TensorLayer's Layers.")

layer = layers[0]

all_params = []

all_layers = []

all_drop = {}

for l in layers:

all_params.extend(l.all_params)

all_layers.extend(l.all_layers)

all_drop.update(l.all_drop)

layer.all_params = list(all_params)

layer.all_layers = list(all_layers)

layer.all_drop = dict(all_drop)

layer.all_layers = list_remove_repeat(layer.all_layers)

layer.all_params = list_remove_repeat(layer.all_params)

"""Initialize the global variables of TensorFlow.

assert sess is not None

# try: # TF12+

sess.run(tf.global_variables_initializer())

# except: # TF11

"""The basic :class:`Layer` class represents a single layer of a neural network.

# Added to allow auto-completion

inputs = None

outputs = None

all_layers = []

all_params = []

all_drop = {}

@deprecated_alias(layer='prev_layer', end_support_version=1.9) # TODO remove this line for the 1.9 release

def __init__(self, prev_layer, name=None):

if name is None:

raise ValueError('Layer must have a name.')

scope_name = tf.get_variable_scope().name

if scope_name:

name = scope_name + '/' + name

self.name = name

# get all properties of previous layer(s)

if isinstance(prev_layer, Layer): # 1. for normal layer have only 1 input i.e. DenseLayer

# Hint : list(), dict() is pass by value (shallow), without them,

# it is pass by reference.

self.all_layers = list(prev_layer.all_layers)

self.all_params = list(prev_layer.all_params)

self.all_drop = dict(prev_layer.all_drop)

elif isinstance(prev_layer, list): # 2. for layer have multiply inputs i.e. ConcatLayer

self.all_layers = list_remove_repeat(sum([l.all_layers for l in prev_layer], []))

self.all_params = list_remove_repeat(sum([l.all_params for l in prev_layer], []))

self.all_drop = dict(sum([list(l.all_drop.items()) for l in prev_layer], []))

elif isinstance(prev_layer, tf.Tensor):

raise Exception("Please use InputLayer to convert Tensor/Placeholder to TL layer")

elif prev_layer is not None: # tl.models

self.all_layers = list(prev_layer.all_layers)

self.all_params = list(prev_layer.all_params)

self.all_drop = dict(prev_layer.all_drop)

# raise Exception("Unknown layer type %s" % type(prev_layer))

def print_params(self, details=True, session=None):

"""Print all info of parameters in the network"""

for i, p in enumerate(self.all_params):

if details:

try:

# logging.info(" param {:3}: {:15} (mean: {:<18}, median: {:<18}, std: {:<18}) {}".format(i, str(p.eval().shape), p.eval().mean(), np.median(p.eval()), p.eval().std(), p.name))

val = p.eval(session=session)

logging.info(

" param {:3}: {:20} {:15} {} (mean: {:<18}, median: {:<18}, std: {:<18}) ".format(

i, p.name, str(val.shape), p.dtype.name, val.mean(), np.median(val), val.std()

)

)

except Exception as e:

logging.info(str(e))

raise Exception(

"Hint: print params details after tl.layers.initialize_global_variables(sess) or use network.print_params(False)."

)

else:

logging.info(" param {:3}: {:20} {:15} {}".format(i, p.name, str(p.get_shape()), p.dtype.name))

logging.info(" num of params: %d" % self.count_params())

def print_layers(self):

"""Print all info of layers in the network"""

for i, layer in enumerate(self.all_layers):

# logging.info(" layer %d: %s" % (i, str(layer)))

logging.info(

" layer {:3}: {:20} {:15} {}".format(i, layer.name, str(layer.get_shape()), layer.dtype.name)

)

def count_params(self):

"""Return the number of parameters in the network"""

n_params = 0

for _i, p in enumerate(self.all_params):

n = 1

# for s in p.eval().shape:

for s in p.get_shape():

try:

s = int(s)

except Exception:

s = 1

if s:

n = n * s

n_params = n_params + n

return n_params

def __str__(self):

return " Last layer is: %s (%s) %s" % (self.__class__.__name__, self.name, self.outputs.get_shape().as_list())

def __getitem__(self, key):

net_new = Layer(prev_layer=None, name=self.name)

net_new.inputs = self.inputs

net_new.outputs = self.outputs[key]

net_new.all_layers = list(self.all_layers[:-1])

net_new.all_layers.append(net_new.outputs)

net_new.all_params = list(self.all_params)

net_new.all_drop = dict(self.all_drop)

return net_new

def __setitem__(self, key, item):

# self.outputs[key] = item

raise NotImplementedError("%s: __setitem__" % self.name)

def __delitem__(self, key):

raise NotImplementedError("%s: __delitem__" % self.name)

def __iter__(self):

for x in self.all_layers:

yield x

def __len__(self):

"""

def __init__(self, inputs=None, name='input'):

super(InputLayer, self).__init__(prev_layer=None, name=name)

logging.info("InputLayer %s: %s" % (self.name, inputs.get_shape()))

self.outputs = inputs

self.all_layers = []

self.all_params = []

"""

def __init__(self, inputs=None, depth=None, on_value=None, off_value=None, axis=None, dtype=None, name='input'):

super(OneHotInputLayer, self).__init__(prev_layer=None, name=name)

logging.info("OneHotInputLayer %s: %s" % (self.name, inputs.get_shape()))

# assert depth != None, "depth is not given"

if depth is None:

logging.info(" [*] depth == None the number of output units is undefined")

self.outputs = tf.one_hot(inputs, depth, on_value=on_value, off_value=off_value, axis=axis, dtype=dtype)

self.all_layers = []

self.all_params = []

"""

def __init__(

self,

inputs=None,

train_labels=None,

vocabulary_size=80000,

embedding_size=200,

num_sampled=64,

nce_loss_args=None,

E_init=tf.random_uniform_initializer(minval=-1.0, maxval=1.0),

E_init_args=None,

nce_W_init=tf.truncated_normal_initializer(stddev=0.03),

nce_W_init_args=None,

nce_b_init=tf.constant_initializer(value=0.0),

nce_b_init_args=None,

name='word2vec',

):

if nce_loss_args is None:

nce_loss_args = {}

if E_init_args is None:

E_init_args = {}

if nce_W_init_args is None:

nce_W_init_args = {}

if nce_b_init_args is None:

nce_b_init_args = {}

super(Word2vecEmbeddingInputlayer, self).__init__(prev_layer=None, name=name)

logging.info("Word2vecEmbeddingInputlayer %s: (%d, %d)" % (self.name, vocabulary_size, embedding_size))

self.inputs = inputs

# Look up embeddings for inputs.

# Note: a row of 'embeddings' is the vector representation of a word.

# for the sake of speed, it is better to slice the embedding matrix

# instead of transfering a word id to one-hot-format vector and then

# multiply by the embedding matrix.

# embed is the outputs of the hidden layer (embedding layer), it is a

# row vector with 'embedding_size' values.

with tf.variable_scope(name):

embeddings = tf.get_variable(

name='embeddings', shape=(vocabulary_size, embedding_size), initializer=E_init,

dtype=LayersConfig.tf_dtype, **E_init_args

)

embed = tf.nn.embedding_lookup(embeddings, self.inputs)

# Construct the variables for the NCE loss (i.e. negative sampling)

nce_weights = tf.get_variable(

name='nce_weights', shape=(vocabulary_size, embedding_size), initializer=nce_W_init,

dtype=LayersConfig.tf_dtype, **nce_W_init_args

)

nce_biases = tf.get_variable(

name='nce_biases', shape=(vocabulary_size), initializer=nce_b_init, dtype=LayersConfig.tf_dtype,

**nce_b_init_args

)

# Compute the average NCE loss for the batch.

# tf.nce_loss automatically draws a new sample of the negative labels

# each time we evaluate the loss.

self.nce_cost = tf.reduce_mean(

tf.nn.nce_loss(

weights=nce_weights, biases=nce_biases, inputs=embed, labels=train_labels, num_sampled=num_sampled,

num_classes=vocabulary_size, **nce_loss_args

)

)

self.outputs = embed

self.normalized_embeddings = tf.nn.l2_normalize(embeddings, 1)

self.all_layers = [self.outputs]

self.all_params = [embeddings, nce_weights, nce_biases]

"""

def __init__(

self,

inputs=None,

vocabulary_size=80000,

embedding_size=200,

E_init=tf.random_uniform_initializer(-0.1, 0.1),

E_init_args=None,

name='embedding',

):

if E_init_args is None:

E_init_args = {}

super(EmbeddingInputlayer, self).__init__(prev_layer=None, name=name)

logging.info("EmbeddingInputlayer %s: (%d, %d)" % (self.name, vocabulary_size, embedding_size))

self.inputs = inputs

with tf.variable_scope(name):

embeddings = tf.get_variable(

name='embeddings', shape=(vocabulary_size, embedding_size), initializer=E_init,

dtype=LayersConfig.tf_dtype, **E_init_args

)

embed = tf.nn.embedding_lookup(embeddings, self.inputs)

self.outputs = embed

self.all_layers = [self.outputs]

self.all_params = [embeddings]

"""The :class:`AverageEmbeddingInputlayer` averages over embeddings of inputs.

def __init__(

self,

inputs,

vocabulary_size,

embedding_size,

pad_value=0,

embeddings_initializer=tf.random_uniform_initializer(-0.1, 0.1),

embeddings_kwargs=None,

name='average_embedding',

):

super(AverageEmbeddingInputlayer, self).__init__(prev_layer=None, name=name)

logging.info("AverageEmbeddingInputlayer %s: (%d, %d)" % (name, vocabulary_size, embedding_size))

# if embeddings_kwargs is None:

# embeddings_kwargs = {}

if inputs.get_shape().ndims != 2:

raise ValueError('inputs must be of size batch_size * batch_sentence_length')

self.inputs = inputs

with tf.variable_scope(name):

self.embeddings = tf.get_variable(

name='embeddings', shape=(vocabulary_size, embedding_size), initializer=embeddings_initializer,

dtype=LayersConfig.tf_dtype,

**(embeddings_kwargs or {})

# **embeddings_kwargs

) # **(embeddings_kwargs or {}),

word_embeddings = tf.nn.embedding_lookup(

self.embeddings,

self.inputs,

name='word_embeddings',

)

# Zero out embeddings of pad value

masks = tf.not_equal(self.inputs, pad_value, name='masks')

word_embeddings *= tf.cast(

tf.expand_dims(masks, axis=-1),

# tf.float32,

dtype=LayersConfig.tf_dtype,

)

sum_word_embeddings = tf.reduce_sum(word_embeddings, axis=1)

# Count number of non-padding words in each sentence

sentence_lengths = tf.count_nonzero(

masks,

axis=1,

keepdims=True,

# dtype=tf.float32,

dtype=LayersConfig.tf_dtype,

name='sentence_lengths',

)

sentence_embeddings = tf.divide(

sum_word_embeddings,

sentence_lengths + 1e-8, # Add epsilon to avoid dividing by 0

name='sentence_embeddings'

)

self.outputs = sentence_embeddings

self.all_layers = [self.outputs]

self.all_params = [self.embeddings]

"""The :class:`DenseLayer` class is a fully connected layer.

@deprecated_alias(layer='prev_layer', end_support_version=1.9) # TODO remove this line for the 1.9 release

def __init__(

self,

prev_layer,

n_units=100,

act=tf.identity,

W_init=tf.truncated_normal_initializer(stddev=0.1),

b_init=tf.constant_initializer(value=0.0),

W_init_args=None,

b_init_args=None,

name='dense',

):

super(DenseLayer, self).__init__(prev_layer=prev_layer, name=name)

logging.info("DenseLayer %s: %d %s" % (name, n_units, act.__name__))

self.inputs = prev_layer.outputs

self.n_units = n_units

if W_init_args is None:

W_init_args = {}

if b_init_args is None:

b_init_args = {}

if self.inputs.get_shape().ndims != 2:

raise Exception("The input dimension must be rank 2, please reshape or flatten it")

n_in = int(self.inputs.get_shape()[-1])

with tf.variable_scope(name):

W = tf.get_variable(

name='W', shape=(n_in, n_units), initializer=W_init, dtype=LayersConfig.tf_dtype, **W_init_args

)

if b_init is not None:

try:

b = tf.get_variable(

name='b', shape=(n_units), initializer=b_init, dtype=LayersConfig.tf_dtype, **b_init_args

)

except Exception: # If initializer is a constant, do not specify shape.

b = tf.get_variable(name='b', initializer=b_init, dtype=LayersConfig.tf_dtype, **b_init_args)

self.outputs = act(tf.matmul(self.inputs, W) + b)

else:

self.outputs = act(tf.matmul(self.inputs, W))

self.all_layers.append(self.outputs)

if b_init is not None:

self.all_params.extend([W, b])

else: