tutorial_applications.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Application examples https://dcase-repo.github.io/dcase_util/tutorial_applications.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import dcase_util as du\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy.random as rnd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataset initialization https://dcase-repo.github.io/dcase_util/tutorial_applications.html#dataset-initialization\n",
    "\n",
    "This examples uses acoustic scene dataset published for DCASE2013, dataset class to handle this class is delivered with the dcase_utils: `dcase_util.datasets.DCASE2013_Scenes_DevelopmentSet`.\n",
    "\n",
    "Dataset needs to be downloaded first, extracted to disk, and prepared for usage:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I] Acoustic Scene Classification Example / GMM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DictContainer :: Class\n",
      "  audio_source                      : Field recording\n",
      "  audio_type                        : Natural\n",
      "  authors                           : D. Giannoulis, E. Benetos, D. Stowell, and M. D. Plumbley\n",
      "  microphone_model                  : Soundman OKM II Klassik/studio A3 electret microphone\n",
      "  recording_device_model            : Unknown\n",
      "  title                             : IEEE AASP CASA Challenge - Public Dataset for Scene Classification Task\n",
      "  url                               : https://archive.org/details/dcase2013_scene_classification\n",
      "\n",
      "MetaDataContainer :: Class\n",
      "  Filename                          : E:/zanco/TCC/datasets\\DCASE2013-acoustic-scenes-development\\meta.txt \n",
      "  Items                             : 100 \n",
      "  Unique\n",
      "    Files                           : 100 \n",
      "    Scene labels                    : 10 \n",
      "    Event labels                    : 0 \n",
      "    Tags                            : 0 \n",
      "\n",
      "  Scene statistics\n",
      "        Scene label             Count   Identifiers   \n",
      "        --------------------   ------   -----------   \n",
      "        bus                        10             0   \n",
      "        busystreet                 10             0   \n",
      "        office                     10             0   \n",
      "        openairmarket              10             0   \n",
      "        park                       10             0   \n",
      "        quietstreet                10             0   \n",
      "        restaurant                 10             0   \n",
      "        supermarket                10             0   \n",
      "        tube                       10             0   \n",
      "        tubestation                10             0   \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Setup logging\n",
    "du.utils.setup_logging()\n",
    "\n",
    "log = du.ui.FancyLogger()\n",
    "log.title('Acoustic Scene Classification Example / GMM')\n",
    "\n",
    "# Create dataset object and set dataset to be stored under 'data' directory.\n",
    "db = du.datasets.DCASE2013_Scenes_DevelopmentSet(\n",
    "    data_path='E:/zanco/TCC/datasets'\n",
    ")\n",
    "\n",
    "# Initialize dataset (download, extract and prepare it).\n",
    "db.initialize()\n",
    "\n",
    "# Show dataset information\n",
    "db.show()\n",
    "# DictContainer :: Class\n",
    "#   audio_source                      : Field recording\n",
    "#   audio_type                        : Natural\n",
    "#   authors                           : D. Giannoulis, E. Benetos, D. Stowell, and M. D. Plumbley\n",
    "#   microphone_model                  : Soundman OKM II Klassik/studio A3 electret microphone\n",
    "#   recording_device_model            : Unknown\n",
    "#   title                             : IEEE AASP CASA Challenge - Public Dataset for Scene Classification Task\n",
    "#   url                               : https://archive.org/details/dcase2013_scene_classification\n",
    "#\n",
    "# MetaDataContainer :: Class\n",
    "#   Filename                          : data/DCASE2013-acoustic-scenes-development/meta.txt\n",
    "#   Items                             : 100\n",
    "#   Unique\n",
    "#     Files                           : 100\n",
    "#     Scene labels                    : 10\n",
    "#     Event labels                    : 0\n",
    "#     Tags                            : 0\n",
    "#\n",
    "#   Scene statistics\n",
    "#         Scene label             Count\n",
    "#         --------------------   ------\n",
    "#         bus                        10\n",
    "#         busystreet                 10\n",
    "#         office                     10\n",
    "#         openairmarket              10\n",
    "#         park                       10\n",
    "#         quietstreet                10\n",
    "#         restaurant                 10\n",
    "#         supermarket                10\n",
    "#         tube                       10\n",
    "#         tubestation                10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feature extraction https://dcase-repo.github.io/dcase_util/tutorial_applications.html#feature-extraction\n",
    "\n",
    "Usually it is most efficient to extract features for all audio files and store them on disk, rather than extracting them each time when acoustic features are needed. Example how to do this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db.audio_file_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I] Feature Extraction\n",
      "[I] ========================================\n",
      "[I]   bus01.wav\n",
      "[I]   bus02.wav\n",
      "[I]   bus03.wav\n",
      "[I]   bus04.wav\n",
      "[I]   bus05.wav\n",
      "[I]   bus06.wav\n",
      "[I]   bus07.wav\n",
      "[I]   bus08.wav\n",
      "[I]   bus09.wav\n",
      "[I]   bus10.wav\n",
      "[I]   busystreet01.wav\n",
      "[I]   busystreet02.wav\n",
      "[I]   busystreet03.wav\n",
      "[I]   busystreet04.wav\n",
      "[I]   busystreet05.wav\n",
      "[I]   busystreet06.wav\n",
      "[I]   busystreet07.wav\n",
      "[I]   busystreet08.wav\n",
      "[I]   busystreet09.wav\n",
      "[I]   busystreet10.wav\n",
      "[I]   office01.wav\n",
      "[I]   office02.wav\n",
      "[I]   office03.wav\n",
      "[I]   office04.wav\n",
      "[I]   office05.wav\n",
      "[I]   office06.wav\n",
      "[I]   office07.wav\n",
      "[I]   office08.wav\n",
      "[I]   office09.wav\n",
      "[I]   office10.wav\n",
      "[I]   openairmarket01.wav\n",
      "[I]   openairmarket02.wav\n",
      "[I]   openairmarket03.wav\n",
      "[I]   openairmarket04.wav\n",
      "[I]   openairmarket05.wav\n",
      "[I]   openairmarket06.wav\n",
      "[I]   openairmarket07.wav\n",
      "[I]   openairmarket08.wav\n",
      "[I]   openairmarket09.wav\n",
      "[I]   openairmarket10.wav\n",
      "[I]   park01.wav\n",
      "[I]   park02.wav\n",
      "[I]   park03.wav\n",
      "[I]   park04.wav\n",
      "[I]   park05.wav\n",
      "[I]   park06.wav\n",
      "[I]   park07.wav\n",
      "[I]   park08.wav\n",
      "[I]   park09.wav\n",
      "[I]   park10.wav\n",
      "[I]   quietstreet01.wav\n",
      "[I]   quietstreet02.wav\n",
      "[I]   quietstreet03.wav\n",
      "[I]   quietstreet04.wav\n",
      "[I]   quietstreet05.wav\n",
      "[I]   quietstreet06.wav\n",
      "[I]   quietstreet07.wav\n",
      "[I]   quietstreet08.wav\n",
      "[I]   quietstreet09.wav\n",
      "[I]   quietstreet10.wav\n",
      "[I]   restaurant01.wav\n",
      "[I]   restaurant02.wav\n",
      "[I]   restaurant03.wav\n",
      "[I]   restaurant04.wav\n",
      "[I]   restaurant05.wav\n",
      "[I]   restaurant06.wav\n",
      "[I]   restaurant07.wav\n",
      "[I]   restaurant08.wav\n",
      "[I]   restaurant09.wav\n",
      "[I]   restaurant10.wav\n",
      "[I]   supermarket01.wav\n",
      "[I]   supermarket02.wav\n",
      "[I]   supermarket03.wav\n",
      "[I]   supermarket04.wav\n",
      "[I]   supermarket05.wav\n",
      "[I]   supermarket06.wav\n",
      "[I]   supermarket07.wav\n",
      "[I]   supermarket08.wav\n",
      "[I]   supermarket09.wav\n",
      "[I]   supermarket10.wav\n",
      "[I]   tube01.wav\n",
      "[I]   tube02.wav\n",
      "[I]   tube03.wav\n",
      "[I]   tube04.wav\n",
      "[I]   tube05.wav\n",
      "[I]   tube06.wav\n",
      "[I]   tube07.wav\n",
      "[I]   tube08.wav\n",
      "[I]   tube09.wav\n",
      "[I]   tube10.wav\n",
      "[I]   tubestation01.wav\n",
      "[I]   tubestation02.wav\n",
      "[I]   tubestation03.wav\n",
      "[I]   tubestation04.wav\n",
      "[I]   tubestation05.wav\n",
      "[I]   tubestation06.wav\n",
      "[I]   tubestation07.wav\n",
      "[I]   tubestation08.wav\n",
      "[I]   tubestation09.wav\n",
      "[I]   tubestation10.wav\n",
      "[I]   DONE       \n",
      "[I] \n"
     ]
    }
   ],
   "source": [
    "log.section_header('Feature Extraction')\n",
    "\n",
    "# Prepare feature extractor\n",
    "extractor = du.features.MfccStaticExtractor(\n",
    "    fs=44100,\n",
    "    win_length_seconds=0.04,\n",
    "    hop_length_seconds=0.02,\n",
    "    n_mfcc=14\n",
    ")\n",
    "# Define feature storage path\n",
    "feature_storage_path = os.path.join('system_data', 'features')\n",
    "\n",
    "# Make sure path exists\n",
    "du.utils.Path().create(feature_storage_path)\n",
    "\n",
    "# Loop over all audio files in the dataset and extract features for them.\n",
    "for audio_filename in db.audio_files:\n",
    "    # Show some progress\n",
    "    log.line(os.path.split(audio_filename)[1], indent=2)\n",
    "\n",
    "    # Get filename for feature data from audio filename\n",
    "    feature_filename = os.path.join(\n",
    "        feature_storage_path,\n",
    "        os.path.split(audio_filename)[1].replace('.wav', '.cpickle')\n",
    "    )\n",
    "\n",
    "    # Load audio data\n",
    "    audio = du.containers.AudioContainer().load(\n",
    "        filename=audio_filename,\n",
    "        mono=True,\n",
    "        fs=extractor.fs\n",
    "    )\n",
    "\n",
    "    # Extract features and store them into FeatureContainer, and save it to the disk\n",
    "    features = du.containers.FeatureContainer(\n",
    "        filename=feature_filename,\n",
    "        data=extractor.extract(audio.data),\n",
    "        time_resolution=extractor.hop_length_seconds\n",
    "    ).save()\n",
    "\n",
    "log.foot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feature normalization https://dcase-repo.github.io/dcase_util/tutorial_applications.html#feature-normalization\n",
    "\n",
    "In this stage, training material is gone through per cross-validation fold and mean & standard deviation are calculated for acoustic features. These normalization factors are used to normalize feature data before using it in the learning and testing stages.\n",
    "\n",
    "Code:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I] Feature Normalization\n",
      "[I] ========================================\n",
      "[I]   Fold 1\n",
      "[I]   Fold 2\n",
      "[I]   Fold 3\n",
      "[I]   Fold 4\n",
      "[I]   Fold 5\n",
      "[I]   DONE       \n",
      "[I] \n"
     ]
    }
   ],
   "source": [
    "log.section_header('Feature Normalization')\n",
    "\n",
    "# Define normalization data storage path\n",
    "normalization_storage_path = os.path.join('system_data', 'normalization')\n",
    "\n",
    "# Make sure path exists\n",
    "du.utils.Path().create(normalization_storage_path)\n",
    "\n",
    "# Loop over all cross-validation folds and calculate mean and std for the training data\n",
    "for fold in db.folds():\n",
    "    # Show some progress\n",
    "    log.line('Fold {fold:d}'.format(fold=fold), indent=2)\n",
    "\n",
    "    # Get filename for the normalization factors\n",
    "    fold_stats_filename = os.path.join(\n",
    "        normalization_storage_path,\n",
    "        'norm_fold_{fold:d}.cpickle'.format(fold=fold)\n",
    "    )\n",
    "\n",
    "    # Normalizer\n",
    "    normalizer = du.data.Normalizer(filename=fold_stats_filename)\n",
    "\n",
    "    # Loop through all training data\n",
    "    for item in db.train(fold=fold):\n",
    "        # Get feature filename\n",
    "        feature_filename = os.path.join(\n",
    "            feature_storage_path,\n",
    "            os.path.split(item.filename)[1].replace('.wav', '.cpickle')\n",
    "        )\n",
    "\n",
    "        # Load feature matrix\n",
    "        features = du.containers.FeatureContainer().load(\n",
    "            filename=feature_filename\n",
    "        )\n",
    "\n",
    "        # Accumulate statistics\n",
    "        normalizer.accumulate(features.data)\n",
    "\n",
    "    # Finalize and save\n",
    "    normalizer.finalize().save()\n",
    "\n",
    "log.foot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model learning https://dcase-repo.github.io/dcase_util/tutorial_applications.html#model-learning\n",
    "\n",
    "In this stage, training material is gone though per cross-validation fold, and acoustic model is learned and stored."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I] Learning\n",
      "[I] ========================================\n",
      "[I]   Fold 1\n",
      "[I]     [bus]\n",
      "[I]     [busystreet]\n",
      "[I]     [office]\n",
      "[I]     [openairmarket]\n",
      "[I]     [park]\n",
      "[I]     [quietstreet]\n",
      "[I]     [restaurant]\n",
      "[I]     [supermarket]\n",
      "[I]     [tube]\n",
      "[I]     [tubestation]\n",
      "[I]   Fold 2\n",
      "[I]     [bus]\n",
      "[I]     [busystreet]\n",
      "[I]     [office]\n",
      "[I]     [openairmarket]\n",
      "[I]     [park]\n",
      "[I]     [quietstreet]\n",
      "[I]     [restaurant]\n",
      "[I]     [supermarket]\n",
      "[I]     [tube]\n",
      "[I]     [tubestation]\n",
      "[I]   Fold 3\n",
      "[I]     [bus]\n",
      "[I]     [busystreet]\n",
      "[I]     [office]\n",
      "[I]     [openairmarket]\n",
      "[I]     [park]\n",
      "[I]     [quietstreet]\n",
      "[I]     [restaurant]\n",
      "[I]     [supermarket]\n",
      "[I]     [tube]\n",
      "[I]     [tubestation]\n",
      "[I]   Fold 4\n",
      "[I]     [bus]\n",
      "[I]     [busystreet]\n",
      "[I]     [office]\n",
      "[I]     [openairmarket]\n",
      "[I]     [park]\n",
      "[I]     [quietstreet]\n",
      "[I]     [restaurant]\n",
      "[I]     [supermarket]\n",
      "[I]     [tube]\n",
      "[I]     [tubestation]\n",
      "[I]   Fold 5\n",
      "[I]     [bus]\n",
      "[I]     [busystreet]\n",
      "[I]     [office]\n",
      "[I]     [openairmarket]\n",
      "[I]     [park]\n",
      "[I]     [quietstreet]\n",
      "[I]     [restaurant]\n",
      "[I]     [supermarket]\n",
      "[I]     [tube]\n",
      "[I]     [tubestation]\n",
      "[I]   DONE       \n",
      "[I] \n"
     ]
    }
   ],
   "source": [
    "log.section_header('Learning')\n",
    "\n",
    "# Imports\n",
    "from sklearn.mixture import GaussianMixture\n",
    "\n",
    "# Define model data storage path\n",
    "model_storage_path = os.path.join('system_data', 'model')\n",
    "\n",
    "# Make sure path exists\n",
    "du.utils.Path().create(model_storage_path)\n",
    "\n",
    "# Loop over all cross-validation folds and learn acoustic models\n",
    "for fold in db.folds():\n",
    "    # Show some progress\n",
    "    log.line('Fold {fold:d}'.format(fold=fold), indent=2)\n",
    "\n",
    "    # Get model filename\n",
    "    fold_model_filename = os.path.join(\n",
    "        model_storage_path,\n",
    "        'model_fold_{fold:d}.cpickle'.format(fold=fold)\n",
    "    )\n",
    "\n",
    "    # Get filename for the normalizer\n",
    "    fold_stats_filename = os.path.join(\n",
    "        normalization_storage_path,\n",
    "        'norm_fold_{fold:d}.cpickle'.format(fold=fold)\n",
    "    )\n",
    "\n",
    "    # Normalizer\n",
    "    normalizer = du.data.Normalizer().load(filename=fold_stats_filename)\n",
    "\n",
    "    # Collect class wise training data\n",
    "    class_wise_data = {}\n",
    "    for scene_label in db.scene_labels():\n",
    "        class_wise_data[scene_label] = []\n",
    "\n",
    "        # Loop through all training items from specific scene class\n",
    "        for item in db.train(fold=fold).filter(scene_label=scene_label):\n",
    "            # Get feature filename\n",
    "            feature_filename = os.path.join(\n",
    "                feature_storage_path,\n",
    "                os.path.split(item.filename)[1].replace('.wav', '.cpickle')\n",
    "            )\n",
    "\n",
    "            # Load all features.\n",
    "            features = du.containers.FeatureContainer().load(\n",
    "                filename=feature_filename\n",
    "            )\n",
    "\n",
    "            # Normalize features.\n",
    "            normalizer.normalize(features)\n",
    "\n",
    "            # Store feature data.\n",
    "            class_wise_data[scene_label].append(features.data)\n",
    "\n",
    "    # Initialize model container.\n",
    "    model = du.containers.DictContainer(filename=fold_model_filename)\n",
    "\n",
    "    # Loop though all scene classes and train acoustic model for each\n",
    "    for scene_label in db.scene_labels():\n",
    "        # Show some progress\n",
    "        log.line('[{scene_label}]'.format(scene_label=scene_label), indent=4)\n",
    "\n",
    "        # Train acoustic model\n",
    "        model[scene_label] = GaussianMixture(\n",
    "            n_components=8\n",
    "        ).fit(\n",
    "            np.hstack(class_wise_data[scene_label]).T\n",
    "        )\n",
    "\n",
    "    # Save model to the disk\n",
    "    model.save()\n",
    "\n",
    "log.foot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Testing https://dcase-repo.github.io/dcase_util/tutorial_applications.html#testing\n",
    "\n",
    "In this stage, testing material is gone through per cross-validation fold, and scene class is estimated for each test sample.\n",
    "\n",
    "Code:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I] Testing\n",
      "[I] ========================================\n",
      "[I]   Fold 1\n",
      "[I]   Fold 2\n",
      "[I]   Fold 3\n",
      "[I]   Fold 4\n",
      "[I]   Fold 5\n",
      "[I]   DONE       \n",
      "[I] \n"
     ]
    }
   ],
   "source": [
    "log.section_header('Testing')\n",
    "\n",
    "# Define model data storage path\n",
    "results_storage_path = os.path.join('system_data', 'results')\n",
    "\n",
    "# Make sure path exists\n",
    "du.utils.Path().create(results_storage_path)\n",
    "\n",
    "# Loop over all cross-validation folds and test\n",
    "for fold in db.folds():\n",
    "    # Show some progress\n",
    "    log.line('Fold {fold:d}'.format(fold=fold), indent=2)\n",
    "\n",
    "    # Get model filename\n",
    "    fold_model_filename = os.path.join(\n",
    "        model_storage_path,\n",
    "        'model_fold_{fold:d}.cpickle'.format(fold=fold)\n",
    "    )\n",
    "\n",
    "    # Load model\n",
    "    model = du.containers.DictContainer().load(\n",
    "        filename=fold_model_filename\n",
    "    )\n",
    "\n",
    "    # Get filename for the normalizer\n",
    "    fold_stats_filename = os.path.join(\n",
    "        normalization_storage_path,\n",
    "        'norm_fold_{fold:d}.cpickle'.format(fold=fold)\n",
    "    )\n",
    "\n",
    "    # Normalizer\n",
    "    normalizer = du.data.Normalizer().load(filename=fold_stats_filename)\n",
    "\n",
    "    # Get results filename\n",
    "    fold_results_filename = os.path.join(results_storage_path, 'res_fold_{fold:d}.txt'.format(fold=fold))\n",
    "\n",
    "    # Initialize results container\n",
    "    res = du.containers.MetaDataContainer(filename=fold_results_filename)\n",
    "\n",
    "    # Loop through all test files from the current cross-validation fold\n",
    "    for item in db.test(fold=fold):\n",
    "        # Get feature filename\n",
    "        feature_filename = os.path.join(\n",
    "            feature_storage_path,\n",
    "            os.path.split(item.filename)[1].replace('.wav', '.cpickle')\n",
    "        )\n",
    "\n",
    "        # Load all features.\n",
    "        features = du.containers.FeatureContainer().load(\n",
    "            filename=feature_filename\n",
    "        )\n",
    "\n",
    "        # Normalize features.\n",
    "        normalizer.normalize(features)\n",
    "\n",
    "        # Initialize log likelihoods matrix\n",
    "        logls = -np.inf * np.ones((db.scene_label_count(), features.frames))\n",
    "\n",
    "        # Loop through all scene classes and get likelihood for each per frame\n",
    "        for scene_label_id, scene_label in enumerate(db.scene_labels()):\n",
    "            logls[scene_label_id] = model[scene_label].score_samples(features.data.T)\n",
    "\n",
    "        # Accumulate log likelihoods\n",
    "        accumulated_logls = du.data.ProbabilityEncoder().collapse_probabilities(\n",
    "            probabilities=logls,\n",
    "            operator='sum'\n",
    "        )\n",
    "\n",
    "        # Estimate scene label based on max likelihood.\n",
    "        estimated_scene_label = du.data.ProbabilityEncoder(\n",
    "            label_list=db.scene_labels()\n",
    "        ).max_selection(\n",
    "            probabilities=accumulated_logls\n",
    "        )\n",
    "\n",
    "        # Store result into results container\n",
    "        res.append(\n",
    "            {\n",
    "                'filename': item.filename,\n",
    "                'scene_label': estimated_scene_label\n",
    "            }\n",
    "        )\n",
    "\n",
    "    # Save results container\n",
    "    res.save()\n",
    "log.foot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluation https://dcase-repo.github.io/dcase_util/tutorial_applications.html#evaluation\n",
    "\n",
    "In this stage, system output is evaluated against ground truth delivered with the dataset.\n",
    "\n",
    "Code:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I] Evaluation\n",
      "[I] ========================================\n",
      "[I]   Scene                | Fold 1   Fold 2   Fold 3   Fold 4   Fold 5 | Average   \n",
      "[I]   -------------------- | ------   ------   ------   ------   ------ | -------   \n",
      "[I]   bus                  | 100.00   100.00   100.00   100.00   100.00 |  100.00   \n",
      "[I]   busystreet           |  66.67    33.33    33.33   100.00    33.33 |   53.33   \n",
      "[I]   office               |  66.67   100.00   100.00    66.67   100.00 |   86.67   \n",
      "[I]   openairmarket        |  33.33   100.00     0.00    66.67   100.00 |   60.00   \n",
      "[I]   park                 |  33.33    33.33     0.00    33.33    33.33 |   26.67   \n",
      "[I]   quietstreet          |  66.67   100.00    33.33    66.67    66.67 |   66.67   \n",
      "[I]   restaurant           |  66.67     0.00    33.33     0.00    33.33 |   26.67   \n",
      "[I]   supermarket          |  33.33     0.00    33.33     0.00    33.33 |   20.00   \n",
      "[I]   tube                 | 100.00    33.33    66.67    66.67    66.67 |   66.67   \n",
      "[I]   tubestation          |   0.00    66.67    33.33    33.33     0.00 |   26.67   \n",
      "[I]   -------------------- | ------   ------   ------   ------   ------ | -------   \n",
      "[I]   Average              |  56.67    56.67    43.33    53.33    56.67 |   53.33   \n",
      "[I] \n",
      "[I]   DONE       \n",
      "[I] \n"
     ]
    }
   ],
   "source": [
    "log.section_header('Evaluation')\n",
    "\n",
    "# Imports\n",
    "import sed_eval\n",
    "\n",
    "all_res = []\n",
    "overall = []\n",
    "class_wise_results = np.zeros((len(db.folds()), len(db.scene_labels())))\n",
    "for fold in db.folds():\n",
    "    # Get results filename\n",
    "    fold_results_filename = os.path.join(\n",
    "        results_storage_path,\n",
    "        'res_fold_{fold:d}.txt'.format(fold=fold)\n",
    "    )\n",
    "\n",
    "    # Get reference scenes\n",
    "    reference_scene_list = db.eval(fold=fold)\n",
    "    for item_id, item in enumerate(reference_scene_list):\n",
    "        # Modify data for sed_eval\n",
    "        reference_scene_list[item_id]['file'] = item.filename\n",
    "\n",
    "    # Load estimated scenes\n",
    "    estimated_scene_list = du.containers.MetaDataContainer().load(\n",
    "        filename=fold_results_filename\n",
    "    )\n",
    "    for item_id, item in enumerate(estimated_scene_list):\n",
    "        # Modify data for sed_eval\n",
    "        estimated_scene_list[item_id]['file'] = item.filename\n",
    "\n",
    "    # Initialize evaluator\n",
    "    evaluator = sed_eval.scene.SceneClassificationMetrics(scene_labels=db.scene_labels())\n",
    "\n",
    "    # Evaluate estimated against reference.\n",
    "    evaluator.evaluate(\n",
    "        reference_scene_list=reference_scene_list,\n",
    "        estimated_scene_list=estimated_scene_list\n",
    "    )\n",
    "\n",
    "    # Get results\n",
    "    results = du.containers.DictContainer(evaluator.results())\n",
    "\n",
    "    # Store fold-wise results\n",
    "    all_res.append(results)\n",
    "    overall.append(results.get_path('overall.accuracy')*100)\n",
    "\n",
    "    # Get scene class-wise results\n",
    "    class_wise_accuracy = []\n",
    "    for scene_label_id, scene_label in enumerate(db.scene_labels()):\n",
    "        class_wise_accuracy.append(results.get_path(['class_wise', scene_label, 'accuracy', 'accuracy']))\n",
    "        class_wise_results[fold-1, scene_label_id] = results.get_path(\n",
    "            ['class_wise', scene_label, 'accuracy', 'accuracy'])\n",
    "\n",
    "# Form results table\n",
    "cell_data = class_wise_results\n",
    "scene_mean_accuracy = np.mean(cell_data, axis=0).reshape((1, -1))\n",
    "cell_data = np.vstack((cell_data, scene_mean_accuracy))\n",
    "fold_mean_accuracy = np.mean(cell_data, axis=1).reshape((-1, 1))\n",
    "cell_data = np.hstack((cell_data, fold_mean_accuracy))\n",
    "\n",
    "scene_list = db.scene_labels()\n",
    "scene_list.extend(['Average'])\n",
    "cell_data = [scene_list] + (cell_data*100).tolist()\n",
    "\n",
    "column_headers = ['Scene']\n",
    "for fold in db.folds():\n",
    "    column_headers.append('Fold {fold:d}'.format(fold=fold))\n",
    "\n",
    "column_headers.append('Average')\n",
    "\n",
    "log.table(\n",
    "    cell_data=cell_data,\n",
    "    column_headers=column_headers,\n",
    "    column_separators=[0, 5],\n",
    "    row_separators=[10],\n",
    "    indent=2\n",
    ")\n",
    "log.foot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}