vae_definition.py

from keras import objectives, backend as K
from keras.layers import Bidirectional, Dense, Embedding, Input, Lambda, LSTM, RepeatVector, TimeDistributed, Add, GRU, SimpleRNN
from keras.models import Model
from keras.layers import Layer
import keras
from recurrentshop import *
from recurrentshop.cells import LSTMCell, GRUCell, SimpleRNNCell
from keras.layers.merge import Concatenate
from keras.utils import to_categorical

import data_class
from settings import *


class KLDivergenceLayer(Layer):

    """ Identity transform layer that adds KL divergence
    to the final model loss.
    """

    def __init__(self, beta=1.0, prior_mean=0.0, prior_std=1.0, *args, **kwargs):
        self.is_placeholder = True

        self.beta = beta
        self.prior_mean = prior_mean
        self.prior_std = prior_std
        super(KLDivergenceLayer, self).__init__(*args, **kwargs)

    def call(self, inputs):

        mu, log_var = inputs
        prior_log_var = K.log(self.prior_std) * 2
        prior_var = K.square(self.prior_std)
        #kl_batch = self.beta *( - .5 * K.sum(1 + log_var - K.square(mu) - K.exp(log_var), axis=1))
        kl_batch = self.beta * ( - 0.5 * K.sum(1 + log_var - prior_log_var - ((K.square(mu - self.prior_mean) + K.exp(log_var)) / prior_var), axis=1))
        self.add_loss(K.mean(kl_batch), inputs=inputs)
        return inputs

class VAE(object):
    def create(self, 
        input_dim=64, 
        output_dim=64, 
        use_embedding=False, 
        embedding_dim=0, 
        input_length=16, 
        output_length=16, 
        latent_rep_size=256, 
        vae_loss = 'categorical_crossentropy',
        optimizer='Adam', 
        activation='sigmoid', 
        lstm_activation='tanh', 
        lstm_state_activation='tanh', 
        epsilon_std=1.0, 
        epsilon_factor=0.0,
        include_composer_decoder=False,
        num_composers=0, 
        composer_weight=1.0, 
        lstm_size=256, 
        cell_type='LSTM', 
        num_layers_encoder=1, 
        num_layers_decoder=1, 
        bidirectional=False, 
        decode=True, 
        teacher_force=False, 
        learning_rate=0.001, 
        split_lstm_vector=True, 
        history=True, 
        beta=0.01, 
        prior_mean=0.0,
        prior_std=1.0,
        decoder_additional_input=False, 
        decoder_additional_input_dim=0, 
        extra_layer=False, 
        meta_instrument=False, 
        meta_instrument_dim=0, 
        meta_instrument_length=0, 
        meta_instrument_activation='sigmoid', 
        meta_instrument_weight=1.0,
        signature_decoder=False,
        signature_dim=0,
        signature_activation='sigmoid',
        signature_weight=1.0,
        composer_decoder_at_notes_output=False,
        composer_decoder_at_notes_weight=1.0,
        composer_decoder_at_notes_activation='softmax',
        composer_decoder_at_instrument_output=False,
        composer_decoder_at_instrument_weight=1.0,
        composer_decoder_at_instrument_activation='softmax',
        meta_velocity=False,
        meta_velocity_length=0, 
        meta_velocity_activation='sigmoid', 
        meta_velocity_weight=1.0,
        meta_held_notes=False,
        meta_held_notes_length=0, 
        meta_held_notes_activation='softmax', 
        meta_held_notes_weight=1.0,
        meta_next_notes=False,
        meta_next_notes_output_length=16,
        meta_next_notes_weight=1.0,
        meta_next_notes_teacher_force=False,
        activation_before_splitting='tanh'
        ):
        self.encoder = None
        self.decoder = None
        self.composer_decoder = None
        self.autoencoder = None
        self.signature_decoder = None

        self.input_dim=input_dim
        self.output_dim = output_dim
        
        self.decode = decode
        self.input_length = input_length
        self.output_length = output_length
        self.latent_rep_size = latent_rep_size
        self.vae_loss = vae_loss
        self.activation = activation
        self.lstm_activation = lstm_activation
        self.lstm_state_activation = lstm_state_activation
        self.include_composer_decoder = include_composer_decoder
        self.num_composers = num_composers
        self.composer_weight = composer_weight
        self.lstm_size = lstm_size
        self.cell_type = cell_type
        self.num_layers_encoder = num_layers_encoder
        self.num_layers_decoder = num_layers_decoder
        self.bidirectional = bidirectional
        self.teacher_force = teacher_force
        self.use_embedding = use_embedding
        self.embedding_dim = embedding_dim
        self.learning_rate = learning_rate
        self.split_lstm_vector = split_lstm_vector
        self.history = history
        self.beta = beta
        self.prior_mean=prior_mean
        self.prior_std=prior_std
        self.decoder_additional_input = decoder_additional_input
        self.decoder_additional_input_dim = decoder_additional_input_dim
        self.epsilon_std = epsilon_std
        self.epsilon_factor = epsilon_factor
        self.extra_layer = extra_layer
        self.meta_instrument= meta_instrument
        self.meta_instrument_dim= meta_instrument_dim
        self.meta_instrument_length = meta_instrument_length
        self.meta_instrument_activation = meta_instrument_activation
        self.meta_instrument_weight = meta_instrument_weight
        self.meta_velocity=meta_velocity
        self.meta_velocity_length=meta_velocity_length 
        self.meta_velocity_activation=meta_velocity_activation
        self.meta_velocity_weight=meta_velocity_weight
        self.meta_held_notes=meta_held_notes
        self.meta_held_notes_length=meta_held_notes_length 
        self.meta_held_notes_activation=meta_held_notes_activation
        self.meta_held_notes_weight=meta_held_notes_weight
        self.meta_next_notes=meta_next_notes
        self.meta_next_notes_output_length=meta_next_notes_output_length
        self.meta_next_notes_weight=meta_next_notes_weight
        self.meta_next_notes_teacher_force=meta_next_notes_teacher_force
        
        self.signature_decoder = signature_decoder
        self.signature_dim = signature_dim
        self.signature_activation = signature_activation
        self.signature_weight = signature_weight

        self.composer_decoder_at_notes_output = composer_decoder_at_notes_output
        self.composer_decoder_at_notes_weight = composer_decoder_at_notes_weight
        self.composer_decoder_at_notes_activation = composer_decoder_at_notes_activation
        self.composer_decoder_at_instrument_output = composer_decoder_at_instrument_output
        self.composer_decoder_at_instrument_weight = composer_decoder_at_instrument_weight
        self.composer_decoder_at_instrument_activation = composer_decoder_at_instrument_activation

        self.activation_before_splitting = activation_before_splitting

        if optimizer == 'RMSprop': self.optimizer = keras.optimizers.RMSprop(lr=learning_rate)
        if optimizer == 'Adam': self.optimizer = keras.optimizers.Adam(lr=learning_rate)

        assert(self.num_layers_encoder > 0)
        assert(self.num_layers_decoder > 0)
        assert(self.input_length > 0)
        assert(self.output_length > 0)
        assert(self.lstm_size > 0)
        assert(self.latent_rep_size > 0)
        assert(self.beta > 0)
        if self.use_embedding:
            assert(embedding_dim > 0)
        if self.meta_instrument:
            assert(meta_instrument_dim > 0)
            assert(meta_instrument_weight > 0)
        if self.signature_decoder:
            assert(self.signature_dim > 0)
            assert(self.signature_weight > 0)

        if self.composer_decoder_at_notes_output:
            assert(composer_decoder_at_notes_weight > 0)
        if self.composer_decoder_at_instrument_output:
            assert(meta_instrument)
            assert(composer_decoder_at_instrument_weight > 0)

        if self.meta_velocity:
            assert(meta_velocity_weight > 0)
            assert(meta_velocity_length > 0)

        if self.meta_held_notes:
            assert(meta_held_notes_weight > 0)
            assert(meta_held_notes_length > 0)
        if self.meta_next_notes:
            assert(meta_next_notes_weight > 0)
            assert(meta_next_notes_output_length > 0)


        
        if self.use_embedding:
            input_x = Input(shape=(self.input_length,), name='embedding_input')
            x = Embedding(self.input_dim, self.embedding_dim)(input_x)
        else:
            input_x = Input(shape=(self.input_length,self.input_dim), name='notes_input')
            x = input_x

        encoder_input_list = [input_x]
        if self.meta_instrument:
            if self.meta_instrument_length > 0:
                meta_instrument_input = Input(shape=(self.meta_instrument_length, self.meta_instrument_dim), name='meta_instrument_input')
            else:
                meta_instrument_input = Input(shape=(self.meta_instrument_dim,), name='meta_instrument_input')
            encoder_input_list.append(meta_instrument_input)
        else:
            meta_instrument_input = None

        if self.meta_velocity:
            meta_velocity_input = Input(shape=(self.meta_velocity_length,1), name='meta_velocity_input')
            encoder_input_list.append(meta_velocity_input)
        else:
            meta_velocity_input = None

        if self.meta_held_notes:
            meta_held_notes_input = Input(shape=(self.meta_held_notes_length,2), name='meta_held_notes_input')
            encoder_input_list.append(meta_held_notes_input)
        else:
            meta_held_notes_input = None

        encoded = self._build_encoder(x, meta_instrument_input, meta_velocity_input, meta_held_notes_input)
        self.encoder = Model(inputs=encoder_input_list, outputs=encoded)

        
        encoded_input = Input(shape=(self.latent_rep_size,), name='encoded_input')
        
        
        if self.use_embedding:
            input_decoder_x = Input(shape=(self.output_dim,), name='embedding_input_decoder_start')
            #decoder_x = Embedding(self.output_dim, self.output_dim, input_length=1)(input_decoder_x)
            decoder_x = input_decoder_x
        else:
            input_decoder_x = Input(shape=(self.output_dim,), name='input_decoder_start')
            decoder_x = input_decoder_x

        autoencoder_decoder_input_list = [input_decoder_x, encoded]
        decoder_input_list = [input_decoder_x, encoded_input]
        autoencoder_input_list = [input_x, input_decoder_x]
        autoencoder_output_list = []

        
        if self.teacher_force:
            ground_truth_input = Input(shape=(self.output_length, self.output_dim), name='ground_truth_input')
            decoder_input_list.append(ground_truth_input)
            autoencoder_decoder_input_list.append(ground_truth_input)
            autoencoder_input_list.append(ground_truth_input)
        else:
            ground_truth_input = None

        if self.history:
            history_input = Input(shape=(self.latent_rep_size,), name='history_input')
            decoder_input_list.append(history_input)
            autoencoder_decoder_input_list.append(history_input)
            autoencoder_input_list.append(history_input)
        else:
            history_input = None

        if decoder_additional_input:
            decoder_additional_input_layer = Input(shape=(decoder_additional_input_dim,), name='decoder_additional_input')
            decoder_input_list.append(decoder_additional_input_layer )
            autoencoder_decoder_input_list.append(decoder_additional_input_layer )
            autoencoder_input_list.append(decoder_additional_input_layer )
        else:
            decoder_additional_input_layer  = False

        if self.meta_instrument:
            input_decoder_meta_instrument_start = Input(shape=(self.meta_instrument_dim,), name='input_decoder_meta_instrument_start')
            decoder_input_list.append(input_decoder_meta_instrument_start)
            autoencoder_decoder_input_list.append(input_decoder_meta_instrument_start)
            autoencoder_input_list.append(input_decoder_meta_instrument_start)
            autoencoder_input_list.append(meta_instrument_input)
        else:
            input_decoder_meta_instrument_start = None

        if self.meta_velocity:
            input_decoder_meta_velocity_start = Input(shape=(1,), name='input_decoder_meta_velocity_start')
            decoder_input_list.append(input_decoder_meta_velocity_start)
            autoencoder_decoder_input_list.append(input_decoder_meta_velocity_start)
            autoencoder_input_list.append(input_decoder_meta_velocity_start)
            autoencoder_input_list.append(meta_velocity_input)
        else:
            input_decoder_meta_velocity_start = None

        if self.meta_held_notes:
            input_decoder_meta_held_notes_start = Input(shape=(2,), name='input_decoder_meta_held_notes_start')
            decoder_input_list.append(input_decoder_meta_held_notes_start)
            autoencoder_decoder_input_list.append(input_decoder_meta_held_notes_start)
            autoencoder_input_list.append(input_decoder_meta_held_notes_start)
            autoencoder_input_list.append(meta_held_notes_input)
        else:
            input_decoder_meta_held_notes_start = None

        if self.meta_next_notes:
            input_decoder_meta_next_notes_start = Input(shape=(self.output_dim,), name='input_decoder_meta_next_notes_start')
            decoder_input_list.append(input_decoder_meta_next_notes_start)
            autoencoder_input_list.append(input_decoder_meta_next_notes_start)
            autoencoder_decoder_input_list.append(input_decoder_meta_next_notes_start)
            if self.meta_next_notes_teacher_force:
                meta_next_notes_ground_truth_input = Input(shape=(self.meta_next_notes_output_length, self.output_dim), name='meta_next_notes_ground_truth_input')
                decoder_input_list.append(meta_next_notes_ground_truth_input)
                autoencoder_decoder_input_list.append(meta_next_notes_ground_truth_input)
                autoencoder_input_list.append(meta_next_notes_ground_truth_input)
            else:
                meta_next_notes_ground_truth_input = None
        else:
            input_decoder_meta_next_notes_start = None
            meta_next_notes_ground_truth_input = None


        decoded, meta_instrument_output, meta_velocity_output, meta_held_notes_output, meta_next_notes_output = self._build_decoder(decoder_x, encoded_input, ground_truth_input, history_input, decoder_additional_input_layer, input_decoder_meta_instrument_start, input_decoder_meta_velocity_start, input_decoder_meta_held_notes_start, input_decoder_meta_next_notes_start, meta_next_notes_ground_truth_input)

        loss_list = []
        loss_weights_list = []
        sample_weight_modes = []

        loss_weights_list.append(1.0)
        sample_weight_modes.append('temporal')
        loss_list.append(self.vae_loss)
        metrics_list = ['accuracy']

        if self.meta_instrument or self.meta_velocity or self.meta_held_notes or self.meta_next_notes:
            decoder_output = [decoded]
            if self.meta_instrument:
                decoder_output.append(meta_instrument_output)
            if self.meta_velocity:
                decoder_output.append(meta_velocity_output)
            if self.meta_held_notes:
                decoder_output.append(meta_held_notes_output)
            if self.meta_next_notes:
                decoder_output.append(meta_next_notes_output)
        else:
            decoder_output = decoded


        self.decoder = Model(inputs=decoder_input_list, outputs=decoder_output, name='decoder')

        decoder_final_output = self.decoder(autoencoder_decoder_input_list)


        if isinstance(decoder_final_output, list):
            autoencoder_output_list.extend(decoder_final_output)
        else:
            autoencoder_output_list.append(decoder_final_output)

        if self.meta_instrument:
            #dont append meta_instrument since it is already appended previously to the autoencoder output
            loss_list.append('categorical_crossentropy')
            loss_weights_list.append(self.meta_instrument_weight)
            sample_weight_modes.append('None')

        if self.meta_velocity:

            #dont append meta_velocity since it is already appended previously to the autoencoder output
            loss_list.append('mse')
            loss_weights_list.append(self.meta_velocity_weight)
            sample_weight_modes.append('None')

        if self.meta_held_notes:
            #dont append meta_held_notes since it is already appended previously to the autoencoder output
            loss_list.append('categorical_crossentropy')
            loss_weights_list.append(self.meta_held_notes_weight)
            sample_weight_modes.append('None')

        if self.meta_next_notes:
            #dont append meta_next_notes since it is already appended previously to the autoencoder output
            loss_list.append('categorical_crossentropy')
            loss_weights_list.append(self.meta_next_notes_weight)
            sample_weight_modes.append('None')

        
        if self.include_composer_decoder:
            predicted_composer = self._build_composer_decoder(encoded_input)
            self.composer_decoder = Model(encoded_input, predicted_composer, name='composer_decoder')
            autoencoder_output_list.append(self.composer_decoder(encoded))
            loss_list.append('categorical_crossentropy')
            loss_weights_list.append(self.composer_weight)
            sample_weight_modes.append('None')

        if self.signature_decoder:

            predicted_signature = self._build_signature_decoder(encoded_input)
            self.signature_decoder = Model(encoded_input, predicted_signature, name='signature_decoder')
            autoencoder_output_list.append(self.signature_decoder(encoded))
            loss_list.append('mse')
            loss_weights_list.append(self.signature_weight)
            sample_weight_modes.append('None')


        if self.composer_decoder_at_notes_output:

            notes_composer_decoder_input = Input(shape=(self.output_length,self.output_dim), name='notes_composer_decoder_input')
            predicted_composer_2 = self._build_composer_decoder_at_notes_output(notes_composer_decoder_input)
            self.composer_decoder_2 = Model(notes_composer_decoder_input, predicted_composer_2, name='composer_decoder_at_notes')
            if not meta_instrument and not meta_velocity and not meta_held_notes and not meta_next_notes:
                autoencoder_output_list.append(self.composer_decoder_2(decoder_final_output))
            else:
                autoencoder_output_list.append(self.composer_decoder_2(decoder_final_output[0]))
            loss_list.append('categorical_crossentropy')
            loss_weights_list.append(self.composer_decoder_at_notes_weight)
            sample_weight_modes.append('None')

        if self.composer_decoder_at_instrument_output:
            if self.meta_instrument_length > 0:
                meta_instrument_composer_decoder_input = Input(shape=(self.meta_instrument_length, self.meta_instrument_dim), name='meta_instrument_composer_decoder_input')
            else:
                meta_instrument_composer_decoder_input = Input(shape=(self.meta_instrument_dim,), name='meta_instrument_composer_decoder_input')
            predicted_composer_3 = self._build_composer_decoder_at_instrument_output(meta_instrument_composer_decoder_input)
            self.composer_decoder_3 = Model(meta_instrument_composer_decoder_input, predicted_composer_3, name='composer_decoder_at_instruments')
            autoencoder_output_list.append(self.composer_decoder_3(decoder_final_output[1]))
            loss_list.append('categorical_crossentropy')
            loss_weights_list.append(self.composer_decoder_at_instrument_weight)
            sample_weight_modes.append('None')

            

        self.autoencoder = Model(inputs=autoencoder_input_list, outputs=autoencoder_output_list, name='autoencoder')
        self.autoencoder.compile(optimizer=self.optimizer,
                                 loss=loss_list,
                                 loss_weights=loss_weights_list,
                                 sample_weight_mode=sample_weight_modes,
                                 metrics=metrics_list)

    def _build_encoder(self, x, meta_instrument_input=None, meta_velocity_input=None, meta_held_notes_input=None):
        h = x
        if self.bidirectional:

            for layer_no in range(1,self.num_layers_encoder-1):
                if self.cell_type == 'SimpleRNN': h = Bidirectional(SimpleRNN(self.lstm_size, return_sequences=True, activation=self.lstm_activation, name='rnn_' + str(layer_no)), merge_mode='concat')(h)
                if self.cell_type == 'LSTM': h = Bidirectional(LSTM(self.lstm_size, return_sequences=True, activation=self.lstm_activation, name='lstm_' + str(layer_no)), merge_mode='concat')(h)
                if self.cell_type == 'GRU': h = Bidirectional(GRU(self.lstm_size, return_sequences=True, activation=self.lstm_activation, name='gru_' + str(layer_no)), merge_mode='concat')(h)
            if self.cell_type == 'SimpleRNN': h = SimpleRNN(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='rnn_' + str(self.num_layers_encoder))(h)
            if self.cell_type == 'LSTM': h = LSTM(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='lstm_' + str(self.num_layers_encoder))(h)
            if self.cell_type == 'GRU': h = GRU(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='gru_' + str(self.num_layers_encoder))(h)
        else:
            for layer_no in range(1, self.num_layers_encoder):
                if self.cell_type == 'SimpleRNN': h = SimpleRNN(self.lstm_size, return_sequences=True, activation=self.lstm_activation, name='rnn_' + str(layer_no))(h)
                if self.cell_type == 'LSTM': h = LSTM(self.lstm_size, return_sequences=True, activation=self.lstm_activation, name='lstm_' + str(layer_no))(h)
                if self.cell_type == 'GRU': h = GRU(self.lstm_size, return_sequences=True, activation=self.lstm_activation, name='gru_' + str(layer_no))(h)
            if self.cell_type == 'SimpleRNN': h = SimpleRNN(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='rnn_' +str(self.num_layers_encoder))(h)
            if self.cell_type == 'LSTM': h = LSTM(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='lstm_' +str(self.num_layers_encoder))(h)
            if self.cell_type == 'GRU': h = GRU(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='gru_' +str(self.num_layers_encoder))(h)
        #h = Dense(self.lstm_size, activation='relu', name='dense_1')(h)

        if self.meta_instrument:
            if self.cell_type == 'SimpleRNN': m = SimpleRNN(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='rnn_meta_instrument')(meta_instrument_input)
            if self.cell_type == 'LSTM': m = LSTM(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='lstm_meta_instrument')(meta_instrument_input)
            if self.cell_type == 'GRU': m = GRU(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='gru_meta_instrument')(meta_instrument_input)
            h = Concatenate(name='concatenated_instrument_and_notes_layer')([h, m])
            
        if self.meta_velocity:
            if self.cell_type == 'SimpleRNN': m = SimpleRNN(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='rnn_meta_velocity')(meta_velocity_input)
            if self.cell_type == 'LSTM': m = LSTM(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='lstm_meta_velocity')(meta_velocity_input)
            if self.cell_type == 'GRU': m = GRU(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='gru_meta_velocity')(meta_velocity_input)
            h = Concatenate(name='concatenated_velocity_and_rest_layer')([h, m])
            
        if self.meta_held_notes:
            if self.cell_type == 'SimpleRNN': m = SimpleRNN(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='rnn_meta_held_notes')(meta_held_notes_input)
            if self.cell_type == 'LSTM': m = LSTM(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='lstm_meta_held_notes')(meta_held_notes_input)
            if self.cell_type == 'GRU': m = GRU(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='gru_meta_held_notes')(meta_held_notes_input)
            h = Concatenate(name='concatenated_meta_held_notes_and_rest_layer')([h, m])

        #use a dense layer to pack all the meta information + notes together
        if self.meta_instrument or self.meta_velocity or self.meta_instrument:
            h = Dense(self.lstm_size, name='extra_instrument_after_concat_layer', activation=self.activation_before_splitting, kernel_initializer='glorot_uniform')(h)

        if self.extra_layer:
            h = Dense(self.lstm_size, name='extra_layer', activation=self.activation_before_splitting, kernel_initializer='glorot_uniform')(h)

        if self.split_lstm_vector:
            half_size = int(self.lstm_size/2)
            h_1 = Lambda(lambda x : x[:,:half_size], output_shape=(half_size,))(h)
            h_2 = Lambda(lambda x : x[:,half_size:], output_shape=(self.lstm_size-half_size,))(h)

        else:
            h_1 = h
            h_2 = h

        def sampling(args):
            z_mean_, z_log_var_ = args
            batch_size = K.shape(z_mean_)[0]
            epsilon = K.random_normal(shape=(batch_size, self.latent_rep_size), mean=0., stddev=self.epsilon_std)
            return z_mean_ + K.exp(z_log_var_ / 2) * epsilon 

        #s_3 = (s_1^2 * s_2^2) / (s_1^2 + s_2^2)
        #tf.contrib.distributions.MultivariateNormalDiag(mean=0.0, stddev=0.05, seed=None)
        z_mean = Dense(self.latent_rep_size, name='z_mean', activation='linear', kernel_initializer='glorot_uniform')(h_1)
        z_log_var = Dense(self.latent_rep_size, name='z_log_var', activation='linear', kernel_initializer='glorot_uniform')(h_2)
        
        if epsilon_factor > 0:
            e = Input(shape=(1,), tensor=K.constant(self.epsilon_factor))
            scaled_z_log_var = Add()[z_log_var, e]
            z_mean, scaled_z_log_var = KLDivergenceLayer(beta=self.beta, prior_mean=self.prior_mean, prior_std=self.prior_std, name='kl_layer')([z_mean, scaled_z_log_var])
        else:
            z_mean, z_log_var = KLDivergenceLayer(beta=self.beta, prior_mean=self.prior_mean, prior_std=self.prior_std, name='kl_layer')([z_mean, z_log_var])
        z = Lambda(sampling, output_shape=(self.latent_rep_size,), name='lambda')([z_mean, z_log_var])
        return (z)


    def _build_decoder(self, input_layer, encoded, ground_truth, history_input, decoder_additional_input_layer, input_decoder_meta_instrument_start, input_decoder_meta_velocity_start, input_decoder_meta_held_notes_start, input_decoder_meta_next_notes_start, meta_next_notes_ground_truth_input):

        input_states = []
        for layer_no in range(0,self.num_layers_decoder):

            state_c = Input((self.lstm_size,))
            input_states.append(state_c)

            if self.cell_type == 'LSTM':
                state_h = Input((self.lstm_size,))
                input_states.append(state_h)
        
        final_states = []
        lstm_input = input_layer
        for layer_no in range(0,self.num_layers_decoder):
            if self.cell_type == 'SimpleRNN': lstm_output, state1_t = SimpleRNNCell(self.lstm_size)([lstm_input, input_states[layer_no]])
            if self.cell_type == 'LSTM': lstm_output, state1_t, state2_t = LSTMCell(self.lstm_size)([lstm_input, input_states[layer_no*2], input_states[layer_no*2+1]])
            if self.cell_type == 'GRU': lstm_output, state1_t = GRUCell(self.lstm_size)([lstm_input, input_states[layer_no]])
            lstm_input = lstm_output
            final_states.append(state1_t)
            if self.cell_type == 'LSTM':
                final_states.append(state2_t)

        output = Dense(self.output_dim, activation=self.activation)(lstm_output)
        # use learn_mode = 'join', test_mode = 'viterbi', sparse_target = True (label indice output)
        
        readout_input_sequence = Input((self.output_length,self.output_dim))
        rnn = RecurrentModel(input_layer, output, initial_states=input_states, final_states=final_states, readout_input=readout_input_sequence, teacher_force=self.teacher_force, decode=self.decode, output_length=self.output_length, return_states=False, state_initializer=None, name='notes')

        if self.history:
            new_encoded = Concatenate()([encoded, history_input])
        else:
            new_encoded = encoded

        if self.decoder_additional_input:
            new_encoded = Concatenate()([new_encoded, decoder_additional_input_layer])
        else:
            new_encoded = new_encoded

        initial_states = []
        bias_initializer = 'zeros'
        for layer_no in range(0,self.num_layers_decoder):
        
            
            encoded_c = Dense(self.lstm_size, activation=self.lstm_state_activation, bias_initializer=bias_initializer)(new_encoded)
            initial_states.append(encoded_c)

            if self.cell_type == 'LSTM':
                encoded_h = Dense(self.lstm_size, activation=self.lstm_state_activation, bias_initializer=bias_initializer)(new_encoded)
                initial_states.append(encoded_h)
            
        decoded = rnn(input_layer, initial_state=initial_states, initial_readout=input_layer, ground_truth=ground_truth)

        if self.meta_instrument:
            input_states = []
            
            state_c = Input((self.lstm_size,))
            input_states.append(state_c)

            if self.cell_type == 'LSTM':
                state_h = Input((self.lstm_size,))
                input_states.append(state_h)
            
            final_states = []
            lstm_input = input_decoder_meta_instrument_start
            if self.cell_type == 'SimpleRNN': lstm_output, state1_t = SimpleRNNCell(self.lstm_size)([lstm_input, input_states[0]])
            if self.cell_type == 'LSTM': lstm_output, state1_t, state2_t = LSTMCell(self.lstm_size)([lstm_input, input_states[0], input_states[1]])
            if self.cell_type == 'GRU': lstm_output, state1_t = GRUCell(self.lstm_size)([lstm_input, input_states[0]])
            lstm_input = lstm_output
            final_states.append(state1_t)
            if self.cell_type == 'LSTM':
                final_states.append(state2_t)

            readout_input_sequence = Input((self.meta_instrument_length,self.meta_instrument_dim))
            output = Dense(self.meta_instrument_dim, activation=self.meta_instrument_activation)(lstm_output)
            rnn = RecurrentModel(input_decoder_meta_instrument_start, output, initial_states=input_states, final_states=final_states, readout_input=readout_input_sequence, teacher_force=False, decode=self.decode, output_length=self.meta_instrument_length, return_states=False, state_initializer=None, name='meta_instrument')
            
            initial_states = []
            bias_initializer = 'zeros'

            encoded_c = Dense(self.lstm_size, activation=self.lstm_state_activation, bias_initializer=bias_initializer)(new_encoded)
            initial_states.append(encoded_c)

            if self.cell_type == 'LSTM':
                encoded_h = Dense(self.lstm_size, activation=self.lstm_state_activation, bias_initializer=bias_initializer)(new_encoded)
                initial_states.append(encoded_h)
            meta_instrument_output = rnn(input_decoder_meta_instrument_start, initial_state=initial_states, initial_readout=input_decoder_meta_instrument_start)
        else:
            meta_instrument_output = None


        if self.meta_velocity:
            input_states = []
            
            state_c = Input((self.lstm_size,))
            input_states.append(state_c)

            if self.cell_type == 'LSTM':
                state_h = Input((self.lstm_size,))
                input_states.append(state_h)
            
            final_states = []
            lstm_input = input_decoder_meta_velocity_start
            if self.cell_type == 'SimpleRNN': lstm_output, state1_t = SimpleRNNCell(self.lstm_size)([lstm_input, input_states[0]])
            if self.cell_type == 'LSTM': lstm_output, state1_t, state2_t = LSTMCell(self.lstm_size)([lstm_input, input_states[0], input_states[1]])
            if self.cell_type == 'GRU': lstm_output, state1_t = GRUCell(self.lstm_size)([lstm_input, input_states[0]])
            lstm_input = lstm_output
            final_states.append(state1_t)
            if self.cell_type == 'LSTM':
                final_states.append(state2_t)

            readout_input_sequence = Input((self.meta_velocity_length,1))
            output = Dense(1, activation=self.meta_velocity_activation)(lstm_output)
            rnn = RecurrentModel(input_decoder_meta_velocity_start, output, initial_states=input_states, final_states=final_states, readout_input=readout_input_sequence, teacher_force=False, decode=self.decode, output_length=self.meta_velocity_length, return_states=False, state_initializer=None, name='meta_velocity')
            
            initial_states = []
            bias_initializer = 'zeros'

            encoded_c = Dense(self.lstm_size, activation=self.lstm_state_activation, bias_initializer=bias_initializer)(new_encoded)
            initial_states.append(encoded_c)

            if self.cell_type == 'LSTM':
                encoded_h = Dense(self.lstm_size, activation=self.lstm_state_activation, bias_initializer=bias_initializer)(new_encoded)
                initial_states.append(encoded_h)
            meta_velocity_output = rnn(input_decoder_meta_velocity_start, initial_state=initial_states, initial_readout=input_decoder_meta_velocity_start)
        else:
            meta_velocity_output = None


        if self.meta_held_notes:
            input_states = []
            
            state_c = Input((self.lstm_size,))
            input_states.append(state_c)

            if self.cell_type == 'LSTM':
                state_h = Input((self.lstm_size,))
                input_states.append(state_h)
            
            final_states = []
            lstm_input = input_decoder_meta_held_notes_start
            if self.cell_type == 'SimpleRNN': lstm_output, state1_t = SimpleRNNCell(self.lstm_size)([lstm_input, input_states[0]])
            if self.cell_type == 'LSTM': lstm_output, state1_t, state2_t = LSTMCell(self.lstm_size)([lstm_input, input_states[0], input_states[1]])
            if self.cell_type == 'GRU': lstm_output, state1_t = GRUCell(self.lstm_size)([lstm_input, input_states[0]])
            lstm_input = lstm_output
            final_states.append(state1_t)
            if self.cell_type == 'LSTM':
                final_states.append(state2_t)

            readout_input_sequence = Input((self.meta_held_notes_length,2))
            output = Dense(2, activation=self.meta_held_notes_activation)(lstm_output)
            rnn = RecurrentModel(input_decoder_meta_held_notes_start, output, initial_states=input_states, final_states=final_states, readout_input=readout_input_sequence, teacher_force=False, decode=self.decode, output_length=self.meta_held_notes_length, return_states=False, state_initializer=None, name='meta_held_notes')
            
            initial_states = []
            bias_initializer = 'zeros'

            encoded_c = Dense(self.lstm_size, activation=self.lstm_state_activation, bias_initializer=bias_initializer)(new_encoded)
            initial_states.append(encoded_c)

            if self.cell_type == 'LSTM':
                encoded_h = Dense(self.lstm_size, activation=self.lstm_state_activation, bias_initializer=bias_initializer)(new_encoded)
                initial_states.append(encoded_h)
            meta_held_notes_output = rnn(input_decoder_meta_held_notes_start, initial_state=initial_states, initial_readout=input_decoder_meta_held_notes_start)
        else:
            meta_held_notes_output = None

        if self.meta_next_notes:
            input_states = []
            for layer_no in range(0,self.num_layers_decoder):
            
                state_c = Input((self.lstm_size,))
                input_states.append(state_c)

                if self.cell_type == 'LSTM':
                    state_h = Input((self.lstm_size,))
                    input_states.append(state_h)
            
            final_states = []
            lstm_input = input_decoder_meta_next_notes_start
            for layer_no in range(0,self.num_layers_decoder):
                if self.cell_type == 'SimpleRNN': lstm_output, state1_t = SimpleRNNCell(self.lstm_size)([lstm_input, input_states[layer_no]])
                if self.cell_type == 'LSTM': lstm_output, state1_t, state2_t = LSTMCell(self.lstm_size)([lstm_input, input_states[layer_no*2], input_states[layer_no*2+1]])
                if self.cell_type == 'GRU': lstm_output, state1_t = GRUCell(self.lstm_size)([lstm_input, input_states[layer_no]])
                lstm_input = lstm_output
                final_states.append(state1_t)
                if self.cell_type == 'LSTM':
                    final_states.append(state2_t)

            output = Dense(self.output_dim, activation=self.activation)(lstm_output)
            
            readout_input_sequence = Input((self.meta_next_notes_output_length,self.output_dim))
            rnn = RecurrentModel(input_decoder_meta_next_notes_start, output, initial_states=input_states, final_states=final_states, readout_input=readout_input_sequence, teacher_force=self.meta_next_notes_teacher_force, decode=self.decode, output_length=self.meta_next_notes_output_length, return_states=False, state_initializer=None, name='next_notes')
            
            initial_states = []
            bias_initializer = 'zeros'

            for layer_no in range(0,self.num_layers_decoder):
             
                encoded_c = Dense(self.lstm_size, activation=self.lstm_state_activation, bias_initializer=bias_initializer)(new_encoded)
                initial_states.append(encoded_c)

                if self.cell_type == 'LSTM':
                    encoded_h = Dense(self.lstm_size, activation=self.lstm_state_activation, bias_initializer=bias_initializer)(new_encoded)
                    initial_states.append(encoded_h)
                
            meta_next_notes_output = rnn(input_decoder_meta_next_notes_start, initial_state=initial_states, initial_readout=input_decoder_meta_next_notes_start, ground_truth=meta_next_notes_ground_truth_input)
        else:
            meta_next_notes_output = None

        return decoded, meta_instrument_output, meta_velocity_output, meta_held_notes_output, meta_next_notes_output

    def _build_composer_decoder(self, encoded_rep):
        composer_latent_length = self.num_composers
        h = Lambda(lambda x : x[:,:composer_latent_length], output_shape=(composer_latent_length,))(encoded_rep)
        composer_prediction = Activation('softmax')(h)
        return composer_prediction


    def _build_signature_decoder(self, encoded_rep):
        decoder_latent_length = self.signature_dim
        #add additional offset if the composer already is attached to the first dimensions
        offset = 0
        if self.composer_decoder:
            offset += self.num_composers
        h = Lambda(lambda x : x[:,offset:offset+decoder_latent_length], output_shape=(decoder_latent_length,))(encoded_rep)
        signature_prediction = Activation(self.signature_activation)(h)
        return signature_prediction

    def _build_composer_decoder_at_notes_output(self, composer_notes_input):

        if self.cell_type == 'SimpleRNN': composer_notes_decoder_prediction = SimpleRNN(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='rnn_composer_decoder_at_notes')(composer_notes_input)
        if self.cell_type == 'LSTM': composer_notes_decoder_prediction = LSTM(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='lstm_composer_decoder_at_notes')(composer_notes_input)
        if self.cell_type == 'GRU': composer_notes_decoder_prediction = GRU(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='gru_composer_decoder_at_notes')(composer_notes_input)
        composer_notes_decoder_prediction = Dense(self.num_composers, activation=self.composer_decoder_at_notes_activation)(composer_notes_decoder_prediction)
        return composer_notes_decoder_prediction

    def _build_composer_decoder_at_instrument_output(self, composer_instrument_input):

        if self.cell_type == 'SimpleRNN': composer_instrument_decoder_prediction = SimpleRNN(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='rnn_composer_decoder_at_instrument')(composer_instrument_input)
        if self.cell_type == 'LSTM': composer_instrument_decoder_prediction = LSTM(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='lstm_composer_decoder_at_instrument')(composer_instrument_input)
        if self.cell_type == 'GRU': composer_instrument_decoder_prediction = GRU(self.lstm_size, return_sequences=False, activation=self.lstm_activation, name='gru_composer_decoder_at_instrument')(composer_instrument_input)
        composer_instrument_decoder_prediction = Dense(self.num_composers, activation=self.composer_decoder_at_instrument_activation)(composer_instrument_decoder_prediction)
        return composer_instrument_decoder_prediction



# prerpares encoder input  for a song that is already split
# X: input pitches of shape (num_samples, input_length, different_pitches)
# I: instruments for each voice of shape (max_voices, different_instruments)
# V: velocity information of shape (num_samples, output_length==input_length), values between 0 and 1 when there is no silent note, 1 denotes MAX_VELOCITY
# D: duration information of shape (num_samples, output_length==input_length), values are 1 if a note is held
def prepare_encoder_input_list(X,I,V,D):
    num_samples = X.shape[0]

    #transform duration into categorical
    D_cat = np.zeros((D.shape[0], D.shape[1], 2))
    for sample in range(num_samples):
        for step in range(output_length):
            if D[sample,step] == 0:
                D_cat[sample, step, 0] = 1
            else:
                D_cat[sample, step, 1] = 1
    D = D_cat

    V = np.copy(V) #make a deep copy since it may be changed if you combine_velocity_and_held_notes
    V = np.expand_dims(V, 2)
    if combine_velocity_and_held_notes:
        for sample in range(num_samples):
            for step in range(output_length):
                if D[sample,step, 1] == 1:
                    #assert, that it is held and therefore has no velocity since its not hit
                    assert(V[sample, step,0] == 0)
                    V[sample,step,0] = 1
    

    #tile the meta_instrument as well for every sample
    I = np.tile(np.expand_dims(I, axis=0), (num_samples,1,1))

    if meta_instrument or meta_velocity or meta_held_notes:
        encoder_input_list = [X]
        if meta_instrument:
            encoder_input_list.append(I)
        if meta_velocity:
            encoder_input_list.append(V)
        if meta_held_notes:
            encoder_input_list.append(D)

        return encoder_input_list
    else:
        return X


# prerpares autoencoder input and output for a song that is already split
# R: latent list of shape (num_samples, latent_dim)
# C: class of epoch, integer in range(num_classes)
# S: normalized signature vector of shape (num_samples, signature_dim)
# H: history list of shape (num_samples, latent_dim), if None will form automatically from R by rolling once
def prepare_decoder_input(R,C,S,H=None):

    num_samples = R.shape[0]

    Y_start = np.zeros((num_samples, output_dim))
    input_list = [Y_start, R]

    if teacher_force:
        empty_Y = np.zeros((num_samples, input_length, output_dim))
        input_list.append(empty_Y)

    if history:
        if H is not None:
            input_list.append(H)
        else:
            history_list = np.zeros(R.shape)
            history_list[1:] = R[:-1]
            input_list.append(history_list)

    if decoder_additional_input:
        decoder_additional_input_list = []
        if decoder_input_composer:
            decoder_additional_input_list.extend(C)
        
        if append_signature_vector_to_latent:
            if len(decoder_additional_input_list) > 0:
                decoder_additional_input_list = np.asarray(decoder_additional_input_list)
                decoder_additional_input_list = np.append(decoder_additional_input_list, S, axis=1)
            else:
                decoder_additional_input_list.extend(S)
        decoder_additional_input_list = np.asarray(decoder_additional_input_list)
        input_list.append(decoder_additional_input_list)

    if meta_instrument:
        meta_instrument_start = np.zeros((num_samples, meta_instrument_dim))
        input_list.append(meta_instrument_start)

    if meta_velocity:
        meta_velocity_start = np.zeros((num_samples,))
        input_list.append(meta_velocity_start)

    if meta_held_notes:
        meta_held_notes_start = np.zeros((num_samples,2))
        input_list.append(meta_held_notes_start)

    if meta_next_notes:
        meta_next_notes_start = np.zeros((num_samples,output_dim))
        input_list.append(meta_next_notes_start)

    return input_list





# prerpares autoencoder input and output for a song that is already split
# X: input pitches of shape (num_samples, input_length, different_pitches)
# Y: ouput pitches of shape (num_samples, ouput_length, different_pitches)
# C: class of epoch, integer in range(num_classes)
# I: instruments for each voice of shape (max_voices, different_instruments)
# V: velocity information of shape (num_samples, output_length==input_length), values between 0 and 1 when there is no silent note, 1 denotes MAX_VELOCITY
# D: duration information of shape (num_samples, output_length==input_length), values are 1 if a note is held
# S: normalized signature vector of shape (num_samples, signature_dim)
# H: history list of shape (num_samples, latent_dim)
def prepare_autoencoder_input_and_output_list(X,Y,C,I,V,D,S,H, return_sample_weight=False):

    num_samples = X.shape[0]

    #transform duration into categorical
    D_cat = np.zeros((D.shape[0], D.shape[1], 2))
    for sample in range(num_samples):
        for step in range(output_length):
            if D[sample,step] == 0:
                D_cat[sample, step, 0] = 1
            else:
                D_cat[sample, step, 1] = 1
    D = D_cat

    V = np.copy(V) #make a deep copy since it may be changed if you combine_velocity_and_held_notes
    V = np.expand_dims(V, 2)
    if combine_velocity_and_held_notes:
        for sample in range(num_samples):
            for step in range(output_length):
                if D[sample,step, 1] == 1:
                    #assert, that it is held and therefore has no velocity since its not hit
                    assert(V[sample, step,0] == 0)
                    V[sample,step,0] = 1
    
    #since we have to predict the last 
    if meta_next_notes:
        N = Y[1:] #next input
        X = X[:-1]
        Y = Y[:-1]
        V = V[:-1]
        D = D[:-1]
        S = S[:-1]
        H = H[:-1]
        num_samples = X.shape[0]

    #create start symbol for every sample
    Y_start = np.zeros((num_samples, Y.shape[2]))

    #transform C into categorical format as well and duplicate it by num_samples
    C = np.asarray([to_categorical(C, num_classes=num_classes)]*num_samples).squeeze()

    #tile the meta_instrument as well for every sample
    meta_instrument_input = np.tile(np.expand_dims(I, axis=0), (num_samples,1,1))

    input_list = [X,Y_start]

    output_list = [Y]

    if return_sample_weight:
        #weight matrix for every sample and steps
        sample_weight = np.ones((num_samples,output_length))
        if include_silent_note:
            #set the weight to silent_weight for every sample where a silent note is played
            sample_weight[np.where(Y[:,:,-1]==1)] = silent_weight

        if include_composer_decoder:
            sample_weight_composer_decoder = np.ones((num_samples,))
            if isinstance(sample_weight, list):
                sample_weight.append(sample_weight_composer_decoder)
            else:
                sample_weight = [sample_weight, sample_weight_composer_decoder]

        if signature_decoder:
            sample_weight_signature_decoder = np.ones((num_samples,))
            if isinstance(sample_weight, list):
                sample_weight.append(sample_weight_signature_decoder)
            else:
                sample_weight = [sample_weight, sample_weight_signature_decoder]

        if composer_decoder_at_notes_output:
            sample_weight_composer_notes_decoder = np.ones((num_samples,))
            if isinstance(sample_weight, list):
                sample_weight.append(sample_weight_composer_notes_decoder)
            else:
                sample_weight = [sample_weight, sample_weight_composer_notes_decoder]

        if composer_decoder_at_instrument_output:
            sample_weight_composer_instrument_decoder = np.ones((num_samples,))
            if isinstance(sample_weight, list):
                sample_weight.append(sample_weight_composer_instrument_decoder)
            else:
                sample_weight = [sample_weight, sample_weight_composer_instrument_decoder]

    if teacher_force:
        input_list.append(Y)

    if history:
        input_list.append(H)
    if decoder_additional_input:
        decoder_additional_input_list = []
        if decoder_input_composer:
            decoder_additional_input_list.extend(C)
        if append_signature_vector_to_latent:
            
            if len(decoder_additional_input_list) > 0:
                decoder_additional_input_list = np.asarray(decoder_additional_input_list)
                decoder_additional_input_list = np.append(decoder_additional_input_list, S, axis=1)
            else:
                decoder_additional_input_list.extend(S)
        decoder_additional_input_list = np.asarray(decoder_additional_input_list)
        input_list.append(decoder_additional_input_list)

    if meta_instrument:
        meta_instrument_start = np.zeros((num_samples, meta_instrument_dim))
        input_list.append(meta_instrument_start)
        input_list.append(meta_instrument_input)
        output_list.append(meta_instrument_input)
        if return_sample_weight:
            sample_weight_meta_instrument = np.ones((num_samples,))
            if isinstance(sample_weight, list):
                sample_weight.append(sample_weight_meta_instrument)
            else:
                sample_weight = [sample_weight, sample_weight_meta_instrument]

    if meta_velocity:
        meta_velocity_start = np.zeros((num_samples,))
        input_list.append(meta_velocity_start)
        input_list.append(V)
        output_list.append(V)
        if return_sample_weight:
            sample_weight_meta_velocity = np.ones((num_samples,))
            if isinstance(sample_weight, list):
                sample_weight.append(sample_weight_meta_velocity)
            else:
                sample_weight = [sample_weight, sample_weight_meta_velocity]

    if meta_held_notes:
        meta_held_notes_start = np.zeros((num_samples,2))
        input_list.append(meta_held_notes_start)
        input_list.append(D)
        output_list.append(D)
        if return_sample_weight:
            sample_weight_meta_held_notes = np.ones((num_samples,))
            if isinstance(sample_weight, list):
                sample_weight.append(sample_weight_meta_held_notes)
            else:
                sample_weight = [sample_weight, sample_weight_meta_held_notes]

    if meta_next_notes:
        meta_next_notes_start = np.zeros((num_samples,output_dim))
        input_list.append(meta_next_notes_start)
        output_list.append(N)
        if return_sample_weight:
            #set the weight to silent_weight for every sample where a silent note is played
            sample_weight_meta_next_notes = np.ones((num_samples,))
            if isinstance(sample_weight, list):
                sample_weight.append(sample_weight_meta_next_notes)
            else:
                sample_weight = [sample_weight, sample_weight_meta_next_notes]

    if include_composer_decoder:
        output_list.append(C)

    if signature_decoder:
        output_list.append(S)

    if composer_decoder_at_notes_output:
        output_list.append(C)

    if composer_decoder_at_instrument_output:
        output_list.append(C)

    if return_sample_weight:
        return input_list, output_list, sample_weight
    else:
        return input_list, output_list

#samples a vector by giving an index back that was chosen
def sample_vector(vector, sample_method):
    if np.sum(vector) > 0:
        if sample_method == 'argmax':
            max_index = np.argmax(vector)

        if sample_method == 'choice':
            vector = vector/(np.sum(vector)*1.0)

            vector = np.log(vector) / temperature
            vector = np.exp(vector) / np.sum(np.exp(vector))

            #give it number_of_tries to find a note that is above the cutoff_sample_threshold
            for _ in range(number_of_tries):
                max_index = np.random.choice(len(vector), p=vector)

                if vector[max_index] > cutoff_sample_threshold:
                    break
    else:
        max_index = 0
    return max_index



def sample_notes_prediction(Y, sample_method):
    assert(Y.ndim == 2 or Y.ndim == 3)
    if Y.ndim == 2:
        input_song = Y
    if Y.ndim == 3:
        input_song = []
        for i in range(Y.shape[0]):
            song_part = Y[i]
            input_song.extend(song_part)


    input_song = np.asarray(input_song)

    output_song = np.zeros((input_song.shape[0], high_crop-low_crop))

    for i, step in enumerate(input_song):

        max_index = sample_vector(step, sample_method)

        if include_silent_note and max_index == len(step)-1:
            continue 

        output_song[i, max_index] = 1

    return output_song

def sample_instrument_prediction(I, sample_method):
    if I.ndim > 1:
        output_list = []
        for i in range(I.shape[0]):
            output_list.append(sample_instrument_prediction(I[i], sample_method))
        return np.asarray(output_list)
    else:
        max_index = sample_vector(I, sample_method)
        ret_X = np.zeros(I.shape)
        ret_X[max_index] = 1
        return ret_X

def sample_held_notes_prediction(D, sample_method):
    if D.ndim > 1:
        output_list = []
        for i in range(D.shape[0]):
            pred = sample_held_notes_prediction(D[i], sample_method)
            if isinstance(pred, int):
                output_list.append(pred)
            else:
                output_list.extend(pred)
        return np.asarray(output_list)
    else:
        #go back from categorical format to list format
        max_index = sample_vector(D, sample_method)
        return int(max_index)
        
#returns:
# Y: ouput pitches of shape (num_samples, ouput_length, different_pitches)
# I: instruments for each voice of shape (max_voices, different_instruments)
# V: velocity information of shape (num_samples, output_length==input_length), values between 0 and 1 when there is no silent note, 1 denotes MAX_VELOCITY
# D: duration information of shape (num_samples, output_length==input_length), values are 1 if a note is held
# N: next note pitches of shape (num_samples, output_length, different_pitches)
# sample method: 'argmax' or 'choice'
def process_decoder_outputs(decoder_outputs, sample_method):

    #initalize to None
    Y = None
    I = None
    V = None
    D = None
    N = None

    if isinstance(decoder_outputs, list):
        Y = decoder_outputs[0]
        num_samples = Y.shape[0]
        Y = sample_notes_prediction(Y, sample_method)
        count = 1
        if meta_instrument or meta_velocity or meta_held_notes_output or meta_next_notes:
            meta_instrument_pred = decoder_outputs[count]
            I = sample_instrument_prediction(meta_instrument_pred, sample_method)
            count += 1

        if meta_velocity:
            #set to 0 if silent note or if too low if not meta_held_notes
            meta_velocity_pred = decoder_outputs[count]
            V = np.zeros((Y.shape[0],))
            for sample in range(meta_velocity_pred.shape[0]):
                V[sample*output_length:(sample+1)*output_length] = meta_velocity_pred[sample,:,0]
            for step in range(V.shape[0]):
                #if silent
                if np.sum(Y[step]) == 0:
                    V[step] = 0

            if override_sampled_pitches_based_on_velocity_info:

                for voice in range(max_voices):
                    previous_pitch = -1
                    previous_velocity = 0.0
                    voice_pitch_roll = Y[voice::max_voices]
                    voice_velocity_roll = V[voice::max_voices]
                    for i, (note_vector, velocity) in enumerate(zip(voice_pitch_roll, voice_velocity_roll)):
                        
                        pitch_is_silent = np.sum(note_vector) == 0
                        if pitch_is_silent:
                            pitch = -1
                        else:
                            pitch = np.argmax(note_vector)

                        velocity_is_silent = velocity < velocity_threshold_such_that_it_is_a_played_note
                        
                        if velocity_is_silent:

                            if not pitch_is_silent and previous_pitch > 0 and previous_pitch != pitch:
                                #play it as loud as previous note
                                V[i*max_voices + voice] = previous_velocity
                        else:
                            if pitch_is_silent:
                                #set velocity to 0, since there is no pitch played
                                V[i*max_voices + voice] = 0

                        previous_pitch = pitch
                        if not velocity_is_silent:
                            previous_velocity = velocity
            count += 1

        if meta_held_notes:
            meta_held_notes_pred = decoder_outputs[count]
            D = sample_held_notes_prediction(meta_held_notes_pred, sample_method)
            count += 1

        if meta_next_notes:
            meta_next_notes_pred = decoder_outputs[count]
            N = sample_notes_prediction(meta_next_notes_pred, sample_method)
            count += 1
    else:
        Y = sample_notes_prediction(decoder_outputs, sample_method)

    length_of_song = Y.shape[0]
    
    # set default outputs if not in output of model
    if I is None: 
        I = np.zeros((length_of_song//output_length,max_voices, meta_instrument_dim))
        I[:,0] = 1 # all piano
    if V is None: 
        #set all to half of the max_velocity
        V = np.ones((length_of_song,)) * (velocity_threshold_such_that_it_is_a_played_note + (1.-velocity_threshold_such_that_it_is_a_played_note)*0.5)
    if D is None: 
        D = np.ones((length_of_song,))
        if meta_velocity:
            #if there is velocity information (for every played note) but no held notes:
            #then you can figure out which notes are played or held
            for step in range(length_of_song):
                if V[step] > velocity_threshold_such_that_it_is_a_played_note:
                    D[step] = 0
    if N is None: 
        N = np.zeros(Y.shape) #all silent

    return Y, I, V, D, N

#returns:
# Y: ouput pitches of shape (num_samples, ouput_length, different_pitches)
# I: instruments for each voice of shape (max_voices, different_instruments)
# V: velocity information of shape (num_samples, output_length==input_length), values between 0 and 1 when there is no silent note, 1 denotes MAX_VELOCITY
# D: duration information of shape (num_samples, output_length==input_length), values are 1 if a note is held
# N: next note pitches of shape (num_samples, output_length, different_pitches)
# sample method: 'argmax' or 'choice'
def process_autoencoder_outputs(autoencoder_outputs, sample_method):
    return process_decoder_outputs(autoencoder_outputs, sample_method)