laughter-detection-interactive.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "laughter-detection.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyP39/UAv9dtYlluOAXnYl/J",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jrgillick/laughter-detection/blob/v2/laughter-detection-interactive.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y-tQtobBO6kf",
        "cellView": "form"
      },
      "source": [
        "#@title Install Software\n",
        "\n",
        "!git clone https://github.com/jrgillick/laughter-detection.git\n",
        "\n",
        "%cd laughter-detection/\n",
        "!pip install tgt\n",
        "!pip install pyloudnorm\n",
        "!pip install praatio==3.8.0\n",
        "!pip install tensorboardX==1.9\n",
        "\n",
        "from google.colab import files"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xxyVpPeGgHiL"
      },
      "source": [
        "# **GPU is not enabled by default**\n",
        "\n",
        "In order to load the model without errors:\n",
        "\n",
        "Go to Runtime > Change runtime type > Hardware accelerator > GPU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8MsgjqOpPJZI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "cellView": "form",
        "outputId": "1deaf7a6-fad3-445c-f156-ce972d15cb50"
      },
      "source": [
        "#@title Setup and Load Model\n",
        "\n",
        "import os, sys, pickle, time, librosa, argparse, torch, numpy as np, pandas as pd, scipy\n",
        "from tqdm import tqdm\n",
        "import tgt\n",
        "sys.path.append('./utils/')\n",
        "import laugh_segmenter\n",
        "import models, configs\n",
        "import dataset_utils, audio_utils, data_loaders, torch_utils\n",
        "from tqdm import tqdm\n",
        "from torch import optim, nn\n",
        "from functools import partial\n",
        "from distutils.util import strtobool\n",
        "\n",
        "sample_rate = 8000\n",
        "\n",
        "model_path = 'checkpoints/in_use/resnet_with_augmentation'\n",
        "config = configs.CONFIG_MAP['resnet_with_augmentation']\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print(f\"Using device {device}\")\n",
        "\n",
        "##### Load the Model\n",
        "\n",
        "model = config['model'](dropout_rate=0.0, linear_layer_size=config['linear_layer_size'], filter_sizes=config['filter_sizes'])\n",
        "feature_fn = config['feature_fn']\n",
        "model.set_device(device)\n",
        "\n",
        "if os.path.exists(model_path):\n",
        "    torch_utils.load_checkpoint(model_path+'/best.pth.tar', model)\n",
        "    model.eval()\n",
        "else:\n",
        "    raise Exception(f\"Model checkpoint not found at {model_path}\")"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using device cuda\n",
            "training with dropout=0.0\n",
            "Loading checkpoint at: checkpoints/in_use/resnet_with_augmentation/best.pth.tar\n",
            "Loading checkpoint at step:  60600\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uoFF6je1PNuU",
        "cellView": "form"
      },
      "source": [
        "#@title Upload an audio file\n",
        "\n",
        "uploaded = files.upload()\n",
        "audio_path = list(uploaded.keys())[0]\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "R3LqFPm9biYP"
      },
      "source": [
        "#@title Choose settings and run Laughter Detection\n",
        "\n",
        "#@markdown This will take a minute\n",
        "\n",
        "threshold = 0.5 #@param {type:\"slider\", min:0.1, max:1.0, step:0.1}\n",
        "min_length = 0.2 #@param {type:\"slider\", min:0.1, max:1.0, step:0.1}\n",
        "save_to_audio_files = True #@param {type:\"boolean\"}\n",
        "save_to_textgrid = False #@param {type:\"boolean\"}\n",
        "output_dir = 'laughter_detection_output' #@param {type:\"string\"}\n",
        "\n",
        "\n",
        "##### Load the audio file and features\n",
        "    \n",
        "inference_dataset = data_loaders.SwitchBoardLaughterInferenceDataset(\n",
        "    audio_path=audio_path, feature_fn=feature_fn, sr=sample_rate)\n",
        "\n",
        "collate_fn=partial(audio_utils.pad_sequences_with_labels,\n",
        "                        expand_channel_dim=config['expand_channel_dim'])\n",
        "\n",
        "inference_generator = torch.utils.data.DataLoader(\n",
        "    inference_dataset, num_workers=4, batch_size=8, shuffle=False, collate_fn=collate_fn)\n",
        "\n",
        "\n",
        "##### Make Predictions\n",
        "\n",
        "probs = []\n",
        "for model_inputs, _ in tqdm(inference_generator):\n",
        "    x = torch.from_numpy(model_inputs).float().to(device)\n",
        "    preds = model(x).cpu().detach().numpy().squeeze()\n",
        "    if len(preds.shape)==0:\n",
        "        preds = [float(preds)]\n",
        "    else:\n",
        "        preds = list(preds)\n",
        "    probs += preds\n",
        "probs = np.array(probs)\n",
        "\n",
        "file_length = audio_utils.get_audio_length(audio_path)\n",
        "\n",
        "fps = len(probs)/float(file_length)\n",
        "\n",
        "probs = laugh_segmenter.lowpass(probs)\n",
        "instances = laugh_segmenter.get_laughter_instances(probs, threshold=threshold, min_length=float(min_length), fps=fps)\n",
        "\n",
        "print(); print(\"found %d laughs.\" % (len (instances)))\n",
        "\n",
        "if len(instances) > 0:\n",
        "    full_res_y, full_res_sr = librosa.load(audio_path,sr=44100)\n",
        "    wav_paths = []\n",
        "    maxv = np.iinfo(np.int16).max\n",
        "    \n",
        "    if save_to_audio_files:\n",
        "        if output_dir is None:\n",
        "            raise Exception(\"Need to specify an output directory to save audio files\")\n",
        "        else:\n",
        "            os.system(f\"mkdir -p {output_dir}\")\n",
        "            for index, instance in enumerate(instances):\n",
        "                laughs = laugh_segmenter.cut_laughter_segments([instance],full_res_y,full_res_sr)\n",
        "                wav_path = output_dir + \"/laugh_\" + str(index) + \".wav\"\n",
        "                scipy.io.wavfile.write(wav_path, full_res_sr, (laughs * maxv).astype(np.int16))\n",
        "                wav_paths.append(wav_path)\n",
        "            print(laugh_segmenter.format_outputs(instances, wav_paths))\n",
        "    \n",
        "    if save_to_textgrid:\n",
        "        laughs = [{'start': i[0], 'end': i[1]} for i in instances]\n",
        "        tg = tgt.TextGrid()\n",
        "        laughs_tier = tgt.IntervalTier(name='laughter', objects=[\n",
        "        tgt.Interval(l['start'], l['end'], 'laugh') for l in laughs])\n",
        "        tg.add_tier(laughs_tier)\n",
        "        fname = os.path.splitext(os.path.basename(audio_path))[0]\n",
        "        tgt.write_to_file(tg, os.path.join(output_dir, fname + '_laughter.TextGrid'))\n",
        "\n",
        "        print('Saved laughter segments in {}'.format(\n",
        "            os.path.join(output_dir, fname + '_laughter.TextGrid')))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "7SujoMnTRxjb"
      },
      "source": [
        "#@title Listen to the detected laughter\n",
        "\n",
        "import IPython\n",
        "from IPython.display import Audio\n",
        "detected_laughs = librosa.util.find_files(output_dir)\n",
        "for laugh in detected_laughs:\n",
        "  IPython.display.display(Audio(laugh))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "69cuYnOGfON9"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}