yaroslavvb

function X = lyap2(A, B, C)
%LYAP2  Lyapunov equation solution using eigenvalue decomposition.
%   X = LYAP2(A,C) solves the special form of the Lyapunov matrix
%   equation:
%
%       A*X + X*A' = -C                                                                                                                                 
%                                                                                                                                                       
%   X = LYAP2(A,B,C) solves the general form of the Lyapunov matrix                                                                                     
%   equation:                                                                                                                                           
%                                                                                                                                                       

yaroslavvb

# An example of turning contraction order into sequence of einsum calls

from opt_einsum import helpers as oe_helpers
import opt_einsum as oe

def print_schedule(path, indices, output_subscript, terms):
    """

    Args:
        path: contraction path in einsum optimizer format, ie, [(0,), (2,), (1, 3), (0, 2), (0, 1)]

yaroslavvb

def index_reduce(values, indices, dim):
    """Reduce values by selecting a single element in dimension dim:

     Example below produces rank-2 tensor out of rank-3 values tensor by indexing as follows
     dim=0: values[index[i,j],i,j]
     dim=1: values[i,index[i,j],j]
     dim=2: values[i,j,index[i,j]]

     When all entries of "indices" are equal to p, the result is equivalent to slicing along that dimension.
     dim=0: values[p,:,:]

yaroslavvb

import os, numpy as np, random, torch
np.random.seed(666)
random.seed(666)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
torch.manual_seed(666)
torch.cuda.manual_seed_all(666+int(os.environ.get("RANK", "0")))
torch.set_printoptions(precision=10)

yaroslavvb

def install_pdb_handler():
    """Signals to automatically start pdb:
      1. CTRL+\\ breaks into pdb.
      2. pdb gets launched on exception.
    """

    import signal
    import pdb

    def handler(_signum, _frame):

yaroslavvb

import time
import threading

import torch
import torch.nn as nn


def simple_model(d, n):
    """Creates linear neural network initialized to identity"""
    layers = []

yaroslavvb

d = 3
q = torch.rand(d)
q /= q.sum()   # empirical distribution (soft labels)
theta = torch.randn(d, requires_grad=True)
p = F.softmax(theta, dim=0)
loss = -torch.sum(q*torch.log(p))
g = p - q
H = torch.diag(p) - outer(p)
torch.allclose(H, hessian(loss, theta))

yaroslavvb

def test():
    u.seed_random(1)

    data_width = 3
    targets_width = 2
    batch_size = 3
    dataset = TinyMNIST('/tmp', download=True, data_width=data_width, targets_width=targets_width, dataset_size=batch_size)
    trainloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=False)

    d1 = data_width ** 2   # hidden layer size, visible size, output size

yaroslavvb

import util as u

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
import numpy as np

# todo: make images global

yaroslavvb

import tensorflow as tf
import memory_util
import numpy as np


def memory_timeline_from_nodestats(nodestats):
  lines = []
  for node in nodestats:
    mem = node.memory
    assert(len(mem) == 1), str(node)