ml snippets

batch

def my_get_batches(x, y, size):
    for batch in range(  len(x) // size  ):
        yield x[batch*size:(batch+1)*size], y[batch*size:(batch+1)*size]

# for batch_x, batch_y in my_get_batches(train_x, train_y, batch_size):

data_splitter

def data_splitter(data, where):
    split = int(len(data) * where)
    return data[:split], data[split:]

normalize

def normalize(array):
    minimun = min(array)
    maximum = max(array)
    dist = abs(maximum - minimun)
    return [ (float(i)-minimun)/dist for i in array ]

tf step loader

"""
import tensorflow_loader as tfloader

folder = "layerno_1024"
no_of_epoches = 10

with tf.Session() as sess:
    ...

    epoch_start = tfloader.load_newest(sess, folder)

    for epoch in range(epoch_start, no_of_epoches):

        ...
        tfloader.save(sess, epoch, folder)
        epoch = epoch + 1
"""

import os
import tensorflow as tf

# this variable cant be updated from the outside
default_save_dir = "chapters"


def get_newest_version_epoch_number(folder=default_save_dir):
    if not os.path.isdir(folder):
        return False

    epoch_versions = [int(epoch) for epoch in os.listdir(folder)]

    if len(epoch_versions) == 0:
        return False

    newest_version = max(epoch_versions)

    return newest_version


def load_newest(session, folder=default_save_dir):
    newest_version = get_newest_version_epoch_number(folder)
    if newest_version == False:
        return 0

    return load(session, newest_version, folder) + 1


def load(session, epoch, folder=default_save_dir):
    saver = tf.train.Saver()
    saver.restore(session, os.path.join(folder, str(epoch)) + "/model.ckpt")

    return epoch


def save(session, epoche, folder=default_save_dir):
    path = os.path.join(folder, str(epoche)) + "/model.ckpt"
    if os.path.isdir(folder):
        if not os.path.isdir(path):
            os.makedirs(path)
    else:
        os.makedirs(path)

    saver = tf.train.Saver()
    saver.save(session, path)