From f19019856ab26f1ede9a304eb2d97d8a0614dcf3 Mon Sep 17 00:00:00 2001 From: Mikko Kotila Date: Sun, 21 Apr 2024 10:24:37 +0300 Subject: [PATCH] remove Nadam from automl --- docs/Examples_PyTorch.md | 2 +- docs/Examples_PyTorch_Code.md | 2 +- docs/Examples_Typical.md | 2 +- docs/Examples_Typical_Code.md | 2 +- docs/Hidden_Layers.md | 2 +- ...al Model Hyperparameter Optimization.ipynb | 2 +- ...ion on Keras with Breast Cancer Data.ipynb | 66 +++++++++--------- ...n with Keras for the Iris Prediction.ipynb | 4 +- ...over Best Models from Experiment Log.ipynb | 68 +++++++++---------- talos/autom8/autoparams.py | 4 +- talos/model/normalizers.py | 4 +- talos/templates/params.py | 14 ++-- tests/commands/test_latest.py | 2 +- tests/commands/test_scan.py | 4 +- 14 files changed, 89 insertions(+), 89 deletions(-) diff --git a/docs/Examples_PyTorch.md b/docs/Examples_PyTorch.md index dc8d468e..f8d95a7a 100644 --- a/docs/Examples_PyTorch.md +++ b/docs/Examples_PyTorch.md @@ -138,7 +138,7 @@ return net, net.parameters() ```python p = {'activation':['relu', 'elu'], - 'optimizer': ['Nadam', 'Adam'], + 'optimizer': ['Adagrad', 'Adam'], 'losses': ['logcosh'], 'hidden_layers':[0, 1, 2], 'batch_size': (20, 50, 5), diff --git a/docs/Examples_PyTorch_Code.md b/docs/Examples_PyTorch_Code.md index bc697299..0d1b0b92 100644 --- a/docs/Examples_PyTorch_Code.md +++ b/docs/Examples_PyTorch_Code.md @@ -106,7 +106,7 @@ def breast_cancer(x_train, y_train, x_val, y_val, params): p = {'activation':['relu', 'elu'], - 'optimizer': ['Nadam', 'Adam'], + 'optimizer': ['Adagrad', 'Adam'], 'losses': ['logcosh'], 'hidden_layers':[0, 1, 2], 'batch_size': (20, 50, 5), diff --git a/docs/Examples_Typical.md b/docs/Examples_Typical.md index 8cbe9e94..63396966 100644 --- a/docs/Examples_Typical.md +++ b/docs/Examples_Typical.md @@ -55,7 +55,7 @@ return out, model ### Parameter Dictionary ```python p = {'activation':['relu', 'elu'], - 'optimizer': ['Nadam', 'Adam'], + 'optimizer': ['Adagrad', 'Adam'], 'losses': ['logcosh'], 'hidden_layers':[0, 1, 2], 'batch_size': (20, 50, 5), diff --git a/docs/Examples_Typical_Code.md b/docs/Examples_Typical_Code.md index 72bf4214..11a3528a 100644 --- a/docs/Examples_Typical_Code.md +++ b/docs/Examples_Typical_Code.md @@ -31,7 +31,7 @@ def iris_model(x_train, y_train, x_val, y_val, params): # set the parameter space boundaries p = {'activation':['relu', 'elu'], - 'optimizer': ['Nadam', 'Adam'], + 'optimizer': ['Adagrad', 'Adam'], 'losses': ['categorical_crossentropy'], 'epochs': [100, 200], 'batch_size': [4, 6, 8]} diff --git a/docs/Hidden_Layers.md b/docs/Hidden_Layers.md index e59b9509..60a719e6 100644 --- a/docs/Hidden_Layers.md +++ b/docs/Hidden_Layers.md @@ -18,7 +18,7 @@ When hidden layers are used, `dropout`, `shapes`, `hidden_layers`, and `first_ne ```python p = {'activation':['relu', 'elu'], - 'optimizer': ['Nadam', 'Adam'], + 'optimizer': ['Adagrad', 'Adam'], 'losses': ['logcosh'], 'shapes': ['brick'], # <<< required 'first_neuron': [32, 64], # <<< required diff --git a/examples/Functional Model Hyperparameter Optimization.ipynb b/examples/Functional Model Hyperparameter Optimization.ipynb index 465727a6..a0a8a906 100644 --- a/examples/Functional Model Hyperparameter Optimization.ipynb +++ b/examples/Functional Model Hyperparameter Optimization.ipynb @@ -125,7 +125,7 @@ " 'epochs': [100],\n", " 'dropout': [0],\n", " 'kernel_initializer': ['uniform','normal'],\n", - " 'optimizer': ['Nadam', 'Adam'],\n", + " 'optimizer': ['Adagrad', 'Adam'],\n", " 'losses': ['binary_crossentropy'],\n", " 'activation':['relu', 'elu'],\n", " 'last_activation': ['sigmoid']}" diff --git a/examples/Hyperparameter Optimization on Keras with Breast Cancer Data.ipynb b/examples/Hyperparameter Optimization on Keras with Breast Cancer Data.ipynb index 7121b501..f16245dd 100644 --- a/examples/Hyperparameter Optimization on Keras with Breast Cancer Data.ipynb +++ b/examples/Hyperparameter Optimization on Keras with Breast Cancer Data.ipynb @@ -2,20 +2,21 @@ "cells": [ { "cell_type": "markdown", + "metadata": {}, "source": [ "" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "This notebook is a compliment to the *Hyperparameter Optimization on Keras* article. " - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## Overview\n", "\n", @@ -28,19 +29,20 @@ "3) Defining the Parameter Space Boundaries \n", "\n", "4) Running the Experiment" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## 1. The Required Inputs and Data" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "from tensorflow.keras.models import Sequential\n", "from tensorflow.keras.layers import Dropout, Dense\n", @@ -50,33 +52,33 @@ "import sys\n", "sys.path.insert(0, '/Users/mikko/Documents/GitHub/talos')\n", "import talos" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# then we load the dataset\n", "x, y = talos.templates.datasets.breast_cancer()\n", "\n", "# and normalize every feature to mean 0, std 1\n", "x = talos.utils.rescale_meanzero(x)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## 2. Creating the Keras Model" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# first we have to make sure to input data and params into the function\n", "def breast_cancer_model(x_train, y_train, x_val, y_val, params):\n", @@ -106,20 +108,20 @@ " verbose=0)\n", "\n", " return history, model" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## 3. Defining the Parameter Space Boundary" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# then we can go ahead and set the parameter space\n", "p = {'first_neuron':[9, 10, 11],\n", @@ -128,24 +130,26 @@ " 'epochs': [100],\n", " 'dropout': [0],\n", " 'kernel_initializer': ['uniform','normal'],\n", - " 'optimizer': ['Nadam', 'Adam'],\n", + " 'optimizer': ['Adagrad', 'Adam'],\n", " 'losses': ['binary_crossentropy'],\n", " 'activation':['relu', 'elu'],\n", " 'last_activation': ['sigmoid']}" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## 4. Starting the Experiment" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], "source": [ "# and run the experiment\n", "t = talos.Scan(x=x,\n", @@ -155,11 +159,7 @@ " experiment_name='breast_cancer',\n", " round_limit=50,\n", " disable_progress_bar=True)" - ], - "outputs": [], - "metadata": { - "collapsed": true - } + ] } ], "metadata": { @@ -183,4 +183,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/examples/Hyperparameter Optimization with Keras for the Iris Prediction.ipynb b/examples/Hyperparameter Optimization with Keras for the Iris Prediction.ipynb index 24d22fd1..9caa4322 100644 --- a/examples/Hyperparameter Optimization with Keras for the Iris Prediction.ipynb +++ b/examples/Hyperparameter Optimization with Keras for the Iris Prediction.ipynb @@ -145,7 +145,7 @@ "metadata": {}, "outputs": [], "source": [ - "from tensorflow.keras.optimizers import Adam, Nadam\n", + "from tensorflow.keras.optimizers.legacy import Adam, Adagrad\n", "from tensorflow.keras.activations import softmax\n", "from tensorflow.keras.losses import categorical_crossentropy, logcosh\n", "\n", @@ -154,7 +154,7 @@ " 'batch_size': [2, 3, 4],\n", " 'epochs': [200],\n", " 'dropout': (0, 0.40, 10),\n", - " 'optimizer': [Adam, Nadam],\n", + " 'optimizer': [Adam, Adagrad],\n", " 'loss': ['categorical_crossentropy'],\n", " 'last_activation': ['softmax'],\n", " 'weight_regulizer': [None]}" diff --git a/examples/Recover Best Models from Experiment Log.ipynb b/examples/Recover Best Models from Experiment Log.ipynb index 44cfd52b..cefe837d 100644 --- a/examples/Recover Best Models from Experiment Log.ipynb +++ b/examples/Recover Best Models from Experiment Log.ipynb @@ -2,42 +2,44 @@ "cells": [ { "cell_type": "markdown", + "metadata": {}, "source": [ "" - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "## How to recover best model from experiment log?\n", "Due to system error or other reason where scan_object is no longer available, it's still possible to get best model/s using nothing but the experiment log. In the below notebook you will learn exactly how.\n", "\n" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "import talos\n", "import wrangle\n", "from tensorflow.keras.models import Sequential\n", "from tensorflow.keras.layers import Dense" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "First we'll have to perform the `Scan()` experiment to produce the experiment log. Because the experiment log is stored on local machine, interrupted `Scan()` or other reason will not affect its availability. The experiment log is updated after each permutation; it contains an up-to-date record of the experiment." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# load the data\n", "x, y = talos.templates.datasets.iris()\n", @@ -45,7 +47,7 @@ "\n", "# set the parameter space boundary\n", "p = {'activation':['relu', 'elu'],\n", - " 'optimizer': ['Nadam', 'Adam'],\n", + " 'optimizer': ['Adagrad', 'Adam'],\n", " 'losses': ['logcosh'],\n", " 'shapes': ['brick'],\n", " 'first_neuron': [16, 32, 64, 128],\n", @@ -82,37 +84,37 @@ " experiment_name='minimal_iris',\n", " params=p,\n", " round_limit=10)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Now we can assume the case where we no longer have access to the `scan_object`. In this `Scan(...experiment_name...)` was set to \"reactivate\" so we'll find a folder with that name in the present working directory. Next we have to find out what is the name of the experiment log." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# get the name of the experiment log\n", "!ls -lhtr minimal_iris" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "What you want to do, is get the name of the `.csv` file you want to use, and use it as part of the input for `experiment_log` in the next step." - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "from talos.utils.recover_best_model import recover_best_model\n", "\n", @@ -124,41 +126,39 @@ " input_model=iris_model,\n", " n_models=5,\n", " task='multi_label')" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "Now we can access the cross-validation results:" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "results" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "We can also access the models and make predictions with them:" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "models[0].predict(x_val)" - ], - "outputs": [], - "metadata": {} + ] } ], "metadata": { @@ -182,4 +182,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/talos/autom8/autoparams.py b/talos/autom8/autoparams.py index 3f18cee9..60fa8bc4 100644 --- a/talos/autom8/autoparams.py +++ b/talos/autom8/autoparams.py @@ -1,5 +1,5 @@ import numpy as np -from tensorflow.keras.optimizers.legacy import Adam, Nadam, Adadelta, SGD +from tensorflow.keras.optimizers.legacy import Adam, Adagrad, Adadelta, SGD loss = {'binary': ['binary_crossentropy', 'logcosh'], @@ -122,7 +122,7 @@ def optimizers(self, optimizers='auto'): ''' if optimizers == 'auto': - self._append_params('optimizer', [Adam, Nadam, Adadelta, SGD]) + self._append_params('optimizer', [Adam, Adagrad, Adadelta, SGD]) else: self._append_params('optimizer', optimizers) diff --git a/talos/model/normalizers.py b/talos/model/normalizers.py index 586541ed..7e178d5f 100644 --- a/talos/model/normalizers.py +++ b/talos/model/normalizers.py @@ -12,7 +12,7 @@ def lr_normalizer(lr, optimizer): """ from tensorflow.keras.optimizers.legacy import SGD, Adam, Adadelta, Adagrad, Adamax, RMSprop - from tensorflow.keras.optimizers.legacy import Nadam + from tensorflow.keras.optimizers.legacy import Adagrad from talos.utils.exceptions import TalosModelError if optimizer == Adadelta: @@ -21,7 +21,7 @@ def lr_normalizer(lr, optimizer): lr /= 100.0 elif optimizer == Adam or optimizer == RMSprop: lr /= 1000.0 - elif optimizer == Adamax or optimizer == Nadam: + elif optimizer == Adamax: lr /= 500.0 else: raise TalosModelError(str(optimizer) + " is not supported by lr_normalizer") diff --git a/talos/templates/params.py b/talos/templates/params.py index 2d6826af..b8b023e6 100644 --- a/talos/templates/params.py +++ b/talos/templates/params.py @@ -1,13 +1,13 @@ def titanic(debug=False): - from tensorflow.keras.optimizers.legacy import Adam, Nadam + from tensorflow.keras.optimizers.legacy import Adam, Adagrad # here use a standard 2d dictionary for inputting the param boundaries p = {'lr': (0.5, 5, 10), 'first_neuron': [4, 8, 16], 'batch_size': [20, 30, 40], 'dropout': (0, 0.5, 5), - 'optimizer': [Adam(), Nadam()], + 'optimizer': [Adam(), Adagrad()], 'epochs': [50, 100, 150], 'losses': ['logcosh', 'binary_crossentropy'], 'shapes': ['brick', 'triangle', 0.2], @@ -21,7 +21,7 @@ def titanic(debug=False): 'first_neuron': [4, 8], 'batch_size': [20, 30], 'dropout': [0.2, 0.3], - 'optimizer': [Adam(), Nadam()], + 'optimizer': [Adam(), Adagrad()], 'epochs': [50, 100], 'losses': ['logcosh', 'binary_crossentropy'], 'shapes': ['brick', 'triangle', 0.2], @@ -34,7 +34,7 @@ def titanic(debug=False): def iris(): - from tensorflow.keras.optimizers.legacy import Adam, Nadam + from tensorflow.keras.optimizers.legacy import Adam, Adagrad from tensorflow.keras.losses import logcosh, categorical_crossentropy from tensorflow.keras.activations import relu, elu, softmax @@ -48,7 +48,7 @@ def iris(): 'weight_regulizer': [None], 'emb_output_dims': [None], 'shapes': ['brick', 'triangle', 0.2], - 'optimizer': [Adam, Nadam], + 'optimizer': [Adam, Adagrad], 'losses': [logcosh, categorical_crossentropy], 'activation': [relu, elu], 'last_activation': [softmax]} @@ -58,7 +58,7 @@ def iris(): def breast_cancer(): - from tensorflow.keras.optimizers.legacy import Adam, Nadam, RMSprop + from tensorflow.keras.optimizers.legacy import Adam, Adagrad, RMSprop from tensorflow.keras.losses import logcosh, binary_crossentropy from tensorflow.keras.activations import relu, elu, sigmoid @@ -70,7 +70,7 @@ def breast_cancer(): 'epochs': [50, 100, 150], 'dropout': (0, 0.5, 5), 'shapes': ['brick', 'triangle', 'funnel'], - 'optimizer': [Adam, Nadam, RMSprop], + 'optimizer': [Adam, Adagrad, RMSprop], 'losses': [logcosh, binary_crossentropy], 'activation': [relu, elu], 'last_activation': [sigmoid]} diff --git a/tests/commands/test_latest.py b/tests/commands/test_latest.py index 89933add..8692034d 100644 --- a/tests/commands/test_latest.py +++ b/tests/commands/test_latest.py @@ -13,7 +13,7 @@ def test_latest(): x, y = talos.templates.datasets.iris() p = {'activation': ['relu', 'elu'], - 'optimizer': ['Nadam', 'Adam'], + 'optimizer': ['Adagrad', 'Adam'], 'losses': ['logcosh'], 'shapes': ['brick'], 'first_neuron': [16, 32, 64, 128], diff --git a/tests/commands/test_scan.py b/tests/commands/test_scan.py index a6e5f620..c6dcdd92 100644 --- a/tests/commands/test_scan.py +++ b/tests/commands/test_scan.py @@ -11,7 +11,7 @@ def test_scan(): from tensorflow.keras.models import Sequential p = {'activation': [relu, elu], - 'optimizer': ['Nadam', Adam], + 'optimizer': ['Adagrad', Adam], 'losses': ['logcosh', binary_crossentropy], 'shapes': ['brick', 'funnel', 'triangle'], 'first_neuron': [16], @@ -50,7 +50,7 @@ def iris_model(x_train, y_train, x_val, y_val, params): x, y = talos.templates.datasets.iris() p_for_q = {'activation': ['relu', 'elu'], - 'optimizer': ['Nadam', 'Adam'], + 'optimizer': ['Adagrad', 'Adam'], 'losses': ['logcosh'], 'shapes': ['brick'], 'first_neuron': [16, 32, 64, 128],