diff --git a/docs/Examples_PyTorch.md b/docs/Examples_PyTorch.md
index dc8d468..f8d95a7 100644
--- a/docs/Examples_PyTorch.md
+++ b/docs/Examples_PyTorch.md
@@ -138,7 +138,7 @@ return net, net.parameters()
```python
p = {'activation':['relu', 'elu'],
- 'optimizer': ['Nadam', 'Adam'],
+ 'optimizer': ['Adagrad', 'Adam'],
'losses': ['logcosh'],
'hidden_layers':[0, 1, 2],
'batch_size': (20, 50, 5),
diff --git a/docs/Examples_PyTorch_Code.md b/docs/Examples_PyTorch_Code.md
index bc69729..0d1b0b9 100644
--- a/docs/Examples_PyTorch_Code.md
+++ b/docs/Examples_PyTorch_Code.md
@@ -106,7 +106,7 @@ def breast_cancer(x_train, y_train, x_val, y_val, params):
p = {'activation':['relu', 'elu'],
- 'optimizer': ['Nadam', 'Adam'],
+ 'optimizer': ['Adagrad', 'Adam'],
'losses': ['logcosh'],
'hidden_layers':[0, 1, 2],
'batch_size': (20, 50, 5),
diff --git a/docs/Examples_Typical.md b/docs/Examples_Typical.md
index 8cbe9e9..6339696 100644
--- a/docs/Examples_Typical.md
+++ b/docs/Examples_Typical.md
@@ -55,7 +55,7 @@ return out, model
### Parameter Dictionary
```python
p = {'activation':['relu', 'elu'],
- 'optimizer': ['Nadam', 'Adam'],
+ 'optimizer': ['Adagrad', 'Adam'],
'losses': ['logcosh'],
'hidden_layers':[0, 1, 2],
'batch_size': (20, 50, 5),
diff --git a/docs/Examples_Typical_Code.md b/docs/Examples_Typical_Code.md
index 72bf421..11a3528 100644
--- a/docs/Examples_Typical_Code.md
+++ b/docs/Examples_Typical_Code.md
@@ -31,7 +31,7 @@ def iris_model(x_train, y_train, x_val, y_val, params):
# set the parameter space boundaries
p = {'activation':['relu', 'elu'],
- 'optimizer': ['Nadam', 'Adam'],
+ 'optimizer': ['Adagrad', 'Adam'],
'losses': ['categorical_crossentropy'],
'epochs': [100, 200],
'batch_size': [4, 6, 8]}
diff --git a/docs/Hidden_Layers.md b/docs/Hidden_Layers.md
index e59b950..60a719e 100644
--- a/docs/Hidden_Layers.md
+++ b/docs/Hidden_Layers.md
@@ -18,7 +18,7 @@ When hidden layers are used, `dropout`, `shapes`, `hidden_layers`, and `first_ne
```python
p = {'activation':['relu', 'elu'],
- 'optimizer': ['Nadam', 'Adam'],
+ 'optimizer': ['Adagrad', 'Adam'],
'losses': ['logcosh'],
'shapes': ['brick'], # <<< required
'first_neuron': [32, 64], # <<< required
diff --git a/examples/Functional Model Hyperparameter Optimization.ipynb b/examples/Functional Model Hyperparameter Optimization.ipynb
index 465727a..a0a8a90 100644
--- a/examples/Functional Model Hyperparameter Optimization.ipynb
+++ b/examples/Functional Model Hyperparameter Optimization.ipynb
@@ -125,7 +125,7 @@
" 'epochs': [100],\n",
" 'dropout': [0],\n",
" 'kernel_initializer': ['uniform','normal'],\n",
- " 'optimizer': ['Nadam', 'Adam'],\n",
+ " 'optimizer': ['Adagrad', 'Adam'],\n",
" 'losses': ['binary_crossentropy'],\n",
" 'activation':['relu', 'elu'],\n",
" 'last_activation': ['sigmoid']}"
diff --git a/examples/Hyperparameter Optimization on Keras with Breast Cancer Data.ipynb b/examples/Hyperparameter Optimization on Keras with Breast Cancer Data.ipynb
index 7121b50..f16245d 100644
--- a/examples/Hyperparameter Optimization on Keras with Breast Cancer Data.ipynb
+++ b/examples/Hyperparameter Optimization on Keras with Breast Cancer Data.ipynb
@@ -2,20 +2,21 @@
"cells": [
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
""
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"This notebook is a compliment to the *Hyperparameter Optimization on Keras* article. "
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"## Overview\n",
"\n",
@@ -28,19 +29,20 @@
"3) Defining the Parameter Space Boundaries \n",
"\n",
"4) Running the Experiment"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"## 1. The Required Inputs and Data"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"from tensorflow.keras.models import Sequential\n",
"from tensorflow.keras.layers import Dropout, Dense\n",
@@ -50,33 +52,33 @@
"import sys\n",
"sys.path.insert(0, '/Users/mikko/Documents/GitHub/talos')\n",
"import talos"
- ],
- "outputs": [],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"# then we load the dataset\n",
"x, y = talos.templates.datasets.breast_cancer()\n",
"\n",
"# and normalize every feature to mean 0, std 1\n",
"x = talos.utils.rescale_meanzero(x)"
- ],
- "outputs": [],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"## 2. Creating the Keras Model"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"# first we have to make sure to input data and params into the function\n",
"def breast_cancer_model(x_train, y_train, x_val, y_val, params):\n",
@@ -106,20 +108,20 @@
" verbose=0)\n",
"\n",
" return history, model"
- ],
- "outputs": [],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"## 3. Defining the Parameter Space Boundary"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"# then we can go ahead and set the parameter space\n",
"p = {'first_neuron':[9, 10, 11],\n",
@@ -128,24 +130,26 @@
" 'epochs': [100],\n",
" 'dropout': [0],\n",
" 'kernel_initializer': ['uniform','normal'],\n",
- " 'optimizer': ['Nadam', 'Adam'],\n",
+ " 'optimizer': ['Adagrad', 'Adam'],\n",
" 'losses': ['binary_crossentropy'],\n",
" 'activation':['relu', 'elu'],\n",
" 'last_activation': ['sigmoid']}"
- ],
- "outputs": [],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"## 4. Starting the Experiment"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
"source": [
"# and run the experiment\n",
"t = talos.Scan(x=x,\n",
@@ -155,11 +159,7 @@
" experiment_name='breast_cancer',\n",
" round_limit=50,\n",
" disable_progress_bar=True)"
- ],
- "outputs": [],
- "metadata": {
- "collapsed": true
- }
+ ]
}
],
"metadata": {
@@ -183,4 +183,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
-}
\ No newline at end of file
+}
diff --git a/examples/Hyperparameter Optimization with Keras for the Iris Prediction.ipynb b/examples/Hyperparameter Optimization with Keras for the Iris Prediction.ipynb
index 24d22fd..9caa432 100644
--- a/examples/Hyperparameter Optimization with Keras for the Iris Prediction.ipynb
+++ b/examples/Hyperparameter Optimization with Keras for the Iris Prediction.ipynb
@@ -145,7 +145,7 @@
"metadata": {},
"outputs": [],
"source": [
- "from tensorflow.keras.optimizers import Adam, Nadam\n",
+ "from tensorflow.keras.optimizers.legacy import Adam, Adagrad\n",
"from tensorflow.keras.activations import softmax\n",
"from tensorflow.keras.losses import categorical_crossentropy, logcosh\n",
"\n",
@@ -154,7 +154,7 @@
" 'batch_size': [2, 3, 4],\n",
" 'epochs': [200],\n",
" 'dropout': (0, 0.40, 10),\n",
- " 'optimizer': [Adam, Nadam],\n",
+ " 'optimizer': [Adam, Adagrad],\n",
" 'loss': ['categorical_crossentropy'],\n",
" 'last_activation': ['softmax'],\n",
" 'weight_regulizer': [None]}"
diff --git a/examples/Recover Best Models from Experiment Log.ipynb b/examples/Recover Best Models from Experiment Log.ipynb
index 44cfd52..cefe837 100644
--- a/examples/Recover Best Models from Experiment Log.ipynb
+++ b/examples/Recover Best Models from Experiment Log.ipynb
@@ -2,42 +2,44 @@
"cells": [
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
""
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"## How to recover best model from experiment log?\n",
"Due to system error or other reason where scan_object is no longer available, it's still possible to get best model/s using nothing but the experiment log. In the below notebook you will learn exactly how.\n",
"\n"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"import talos\n",
"import wrangle\n",
"from tensorflow.keras.models import Sequential\n",
"from tensorflow.keras.layers import Dense"
- ],
- "outputs": [],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"First we'll have to perform the `Scan()` experiment to produce the experiment log. Because the experiment log is stored on local machine, interrupted `Scan()` or other reason will not affect its availability. The experiment log is updated after each permutation; it contains an up-to-date record of the experiment."
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"# load the data\n",
"x, y = talos.templates.datasets.iris()\n",
@@ -45,7 +47,7 @@
"\n",
"# set the parameter space boundary\n",
"p = {'activation':['relu', 'elu'],\n",
- " 'optimizer': ['Nadam', 'Adam'],\n",
+ " 'optimizer': ['Adagrad', 'Adam'],\n",
" 'losses': ['logcosh'],\n",
" 'shapes': ['brick'],\n",
" 'first_neuron': [16, 32, 64, 128],\n",
@@ -82,37 +84,37 @@
" experiment_name='minimal_iris',\n",
" params=p,\n",
" round_limit=10)"
- ],
- "outputs": [],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"Now we can assume the case where we no longer have access to the `scan_object`. In this `Scan(...experiment_name...)` was set to \"reactivate\" so we'll find a folder with that name in the present working directory. Next we have to find out what is the name of the experiment log."
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"# get the name of the experiment log\n",
"!ls -lhtr minimal_iris"
- ],
- "outputs": [],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"What you want to do, is get the name of the `.csv` file you want to use, and use it as part of the input for `experiment_log` in the next step."
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"from talos.utils.recover_best_model import recover_best_model\n",
"\n",
@@ -124,41 +126,39 @@
" input_model=iris_model,\n",
" n_models=5,\n",
" task='multi_label')"
- ],
- "outputs": [],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"Now we can access the cross-validation results:"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"results"
- ],
- "outputs": [],
- "metadata": {}
+ ]
},
{
"cell_type": "markdown",
+ "metadata": {},
"source": [
"We can also access the models and make predictions with them:"
- ],
- "metadata": {}
+ ]
},
{
"cell_type": "code",
"execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"models[0].predict(x_val)"
- ],
- "outputs": [],
- "metadata": {}
+ ]
}
],
"metadata": {
@@ -182,4 +182,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
-}
\ No newline at end of file
+}
diff --git a/talos/autom8/autoparams.py b/talos/autom8/autoparams.py
index 3f18cee..60fa8bc 100644
--- a/talos/autom8/autoparams.py
+++ b/talos/autom8/autoparams.py
@@ -1,5 +1,5 @@
import numpy as np
-from tensorflow.keras.optimizers.legacy import Adam, Nadam, Adadelta, SGD
+from tensorflow.keras.optimizers.legacy import Adam, Adagrad, Adadelta, SGD
loss = {'binary': ['binary_crossentropy', 'logcosh'],
@@ -122,7 +122,7 @@ def optimizers(self, optimizers='auto'):
'''
if optimizers == 'auto':
- self._append_params('optimizer', [Adam, Nadam, Adadelta, SGD])
+ self._append_params('optimizer', [Adam, Adagrad, Adadelta, SGD])
else:
self._append_params('optimizer', optimizers)
diff --git a/talos/model/normalizers.py b/talos/model/normalizers.py
index 586541e..7e178d5 100644
--- a/talos/model/normalizers.py
+++ b/talos/model/normalizers.py
@@ -12,7 +12,7 @@ def lr_normalizer(lr, optimizer):
"""
from tensorflow.keras.optimizers.legacy import SGD, Adam, Adadelta, Adagrad, Adamax, RMSprop
- from tensorflow.keras.optimizers.legacy import Nadam
+ from tensorflow.keras.optimizers.legacy import Adagrad
from talos.utils.exceptions import TalosModelError
if optimizer == Adadelta:
@@ -21,7 +21,7 @@ def lr_normalizer(lr, optimizer):
lr /= 100.0
elif optimizer == Adam or optimizer == RMSprop:
lr /= 1000.0
- elif optimizer == Adamax or optimizer == Nadam:
+ elif optimizer == Adamax:
lr /= 500.0
else:
raise TalosModelError(str(optimizer) + " is not supported by lr_normalizer")
diff --git a/talos/templates/params.py b/talos/templates/params.py
index 2d6826a..b8b023e 100644
--- a/talos/templates/params.py
+++ b/talos/templates/params.py
@@ -1,13 +1,13 @@
def titanic(debug=False):
- from tensorflow.keras.optimizers.legacy import Adam, Nadam
+ from tensorflow.keras.optimizers.legacy import Adam, Adagrad
# here use a standard 2d dictionary for inputting the param boundaries
p = {'lr': (0.5, 5, 10),
'first_neuron': [4, 8, 16],
'batch_size': [20, 30, 40],
'dropout': (0, 0.5, 5),
- 'optimizer': [Adam(), Nadam()],
+ 'optimizer': [Adam(), Adagrad()],
'epochs': [50, 100, 150],
'losses': ['logcosh', 'binary_crossentropy'],
'shapes': ['brick', 'triangle', 0.2],
@@ -21,7 +21,7 @@ def titanic(debug=False):
'first_neuron': [4, 8],
'batch_size': [20, 30],
'dropout': [0.2, 0.3],
- 'optimizer': [Adam(), Nadam()],
+ 'optimizer': [Adam(), Adagrad()],
'epochs': [50, 100],
'losses': ['logcosh', 'binary_crossentropy'],
'shapes': ['brick', 'triangle', 0.2],
@@ -34,7 +34,7 @@ def titanic(debug=False):
def iris():
- from tensorflow.keras.optimizers.legacy import Adam, Nadam
+ from tensorflow.keras.optimizers.legacy import Adam, Adagrad
from tensorflow.keras.losses import logcosh, categorical_crossentropy
from tensorflow.keras.activations import relu, elu, softmax
@@ -48,7 +48,7 @@ def iris():
'weight_regulizer': [None],
'emb_output_dims': [None],
'shapes': ['brick', 'triangle', 0.2],
- 'optimizer': [Adam, Nadam],
+ 'optimizer': [Adam, Adagrad],
'losses': [logcosh, categorical_crossentropy],
'activation': [relu, elu],
'last_activation': [softmax]}
@@ -58,7 +58,7 @@ def iris():
def breast_cancer():
- from tensorflow.keras.optimizers.legacy import Adam, Nadam, RMSprop
+ from tensorflow.keras.optimizers.legacy import Adam, Adagrad, RMSprop
from tensorflow.keras.losses import logcosh, binary_crossentropy
from tensorflow.keras.activations import relu, elu, sigmoid
@@ -70,7 +70,7 @@ def breast_cancer():
'epochs': [50, 100, 150],
'dropout': (0, 0.5, 5),
'shapes': ['brick', 'triangle', 'funnel'],
- 'optimizer': [Adam, Nadam, RMSprop],
+ 'optimizer': [Adam, Adagrad, RMSprop],
'losses': [logcosh, binary_crossentropy],
'activation': [relu, elu],
'last_activation': [sigmoid]}
diff --git a/tests/commands/test_latest.py b/tests/commands/test_latest.py
index 89933ad..8692034 100644
--- a/tests/commands/test_latest.py
+++ b/tests/commands/test_latest.py
@@ -13,7 +13,7 @@ def test_latest():
x, y = talos.templates.datasets.iris()
p = {'activation': ['relu', 'elu'],
- 'optimizer': ['Nadam', 'Adam'],
+ 'optimizer': ['Adagrad', 'Adam'],
'losses': ['logcosh'],
'shapes': ['brick'],
'first_neuron': [16, 32, 64, 128],
diff --git a/tests/commands/test_scan.py b/tests/commands/test_scan.py
index a6e5f62..c6dcdd9 100644
--- a/tests/commands/test_scan.py
+++ b/tests/commands/test_scan.py
@@ -11,7 +11,7 @@ def test_scan():
from tensorflow.keras.models import Sequential
p = {'activation': [relu, elu],
- 'optimizer': ['Nadam', Adam],
+ 'optimizer': ['Adagrad', Adam],
'losses': ['logcosh', binary_crossentropy],
'shapes': ['brick', 'funnel', 'triangle'],
'first_neuron': [16],
@@ -50,7 +50,7 @@ def iris_model(x_train, y_train, x_val, y_val, params):
x, y = talos.templates.datasets.iris()
p_for_q = {'activation': ['relu', 'elu'],
- 'optimizer': ['Nadam', 'Adam'],
+ 'optimizer': ['Adagrad', 'Adam'],
'losses': ['logcosh'],
'shapes': ['brick'],
'first_neuron': [16, 32, 64, 128],