004_prompt_checker.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNQHx5oUPL3ScJ7us9hDNuq",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/karaage0703/stable-diffusion-colab-tools/blob/main/004_prompt_checker.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Prompt Checker"
      ],
      "metadata": {
        "id": "AyeX9VhyNB-V"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Quick Check"
      ],
      "metadata": {
        "id": "W6jtu54BaXAS"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Install Library"
      ],
      "metadata": {
        "id": "yFcJtuL9NkBH"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "srk5Ix8aM2WF"
      },
      "outputs": [],
      "source": [
        "!pip -qq install transformers\n",
        "!pip -qq install ftfy regex\n",
        "\n",
        "from transformers import CLIPTokenizer\n",
        "tokenizer = CLIPTokenizer.from_pretrained(\"openai/clip-vit-large-patch14\") "
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Input Prompt"
      ],
      "metadata": {
        "id": "BxGiijhrH4i8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = \"This is an apple\" #@param {type:\"string\"}"
      ],
      "metadata": {
        "id": "auGi-9XrNJZ2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Check token size <= 75 is OK"
      ],
      "metadata": {
        "id": "YpjxL-oGaIAA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "token_size = len(tokenizer.tokenize(prompt))\n",
        "if token_size <= tokenizer.model_max_length - 2:\n",
        "    print('OK: token size is ' + str(token_size))\n",
        "else:\n",
        "    print('NG: token size is ' + str(token_size))"
      ],
      "metadata": {
        "id": "LP1-Sn7UaNJ8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Check token"
      ],
      "metadata": {
        "id": "ce7xX7SOagYA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(tokenizer.tokenize(prompt)[0:tokenizer.model_max_length-2])"
      ],
      "metadata": {
        "id": "bwKSYZVjaDA0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Check in detail"
      ],
      "metadata": {
        "id": "syO_pe_NaT0n"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Use tokenizer"
      ],
      "metadata": {
        "id": "g1Cn02nJIUhj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text_input = tokenizer(prompt)\n",
        "text_input"
      ],
      "metadata": {
        "id": "4o1dJAN3ICDQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Decode. `49406` is start of text, `49407` is End of text"
      ],
      "metadata": {
        "id": "LbERquSaIXvl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(tokenizer.decoder.get(49406))\n",
        "print(tokenizer.decoder.get(589))\n",
        "print(tokenizer.decoder.get(533))\n",
        "print(tokenizer.decoder.get(550))\n",
        "print(tokenizer.decoder.get(3055))\n",
        "print(tokenizer.decoder.get(49407))"
      ],
      "metadata": {
        "id": "sigbw4DvIiBW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Max length of model (-2 means start of text and end of text)"
      ],
      "metadata": {
        "id": "nOkjq2IGJMO3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.model_max_length - 2"
      ],
      "metadata": {
        "id": "f_BLp1QnJKoA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Reference\n",
        "- https://note.com/hugiri/n/n970f9deb55b2\n",
        "- https://note.com/npaka/n/nb08941a36c8b\n",
        "- https://colab.research.google.com/github/tomo-makes/sd-notebooks/blob/master/Grokking_Stable_Diffusion_ja.ipynb"
      ],
      "metadata": {
        "id": "2nSVtXkuM9Pc"
      }
    }
  ]
}