anima_turn_text_to_video.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/saharmor/anima/blob/main/anima_turn_text_to_video.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<center>\n",
        "<img src=\"https://emojipedia-us.s3.dualstack.us-west-1.amazonaws.com/thumbs/240/apple/325/sparkles_2728.png\" width=\"120\">\n",
        "</center>\n",
        "\n",
        "# <center>Anima - Turn text into video</center>\n",
        "###<center>Generate videos from YouTube videos using OpenAI's Whisper, Stable Diffusion, and Google FILM</center>\n",
        "\n",
        "#### <center> [Github Repository](https://github.com/saharmor/anima) </center>\n",
        "##### <center> By [Sahar Mor](https://twitter.com/theaievangelist) and [Abhay Kashyap](https://twitter.com/hayabhay) </center>\n",
        "\n"
      ],
      "metadata": {
        "id": "qLsbRGVXL32v"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown # 1. User configuration - set all the parameters in this cell and run it\n",
        "#@markdown </br>\n",
        "\n",
        "#@markdown ## 1.1 Hugging Face (HF) setup\n",
        "#@markdown Instructions for creating a HF account and token https://huggingface.co/docs/huggingface_hub/quick-start#login\n",
        "\n",
        "#@markdown 👉 **IMPORTANT** Agree to Runway's ToS on HF https://huggingface.co/docs/diffusers/using-diffusers/loading#loading-pipelines-that-require-access-request\n",
        "!git config --global credential.helper store\n",
        "!huggingface-cli login\n",
        "#@markdown </br>\n",
        "\n",
        "#@markdown ## 1.2 Choose YouTube video to transcribe\n",
        "URL = \"https://www.youtube.com/watch?v=t0imaSCnSuA\" #@param {type:\"string\"}\n",
        "start_time = 119 #@param {type:\"integer\"}\n",
        "duration = 15 #@param {type:\"integer\"}\n",
        "#@markdown </br>\n",
        "\n",
        "#@markdown ## 1.3 Prompt fine-tuning\n",
        "\n",
        "prompt_suffix = \"Happy, colorful, hyper-realistic. Highly detailed. High quality.\" #@param {type:\"string\"}\n",
        "#@markdown > Text to append to all prompts. This text will ensure the generated images have the same style. Change this text based on the theme of your generated video. You can drive inspiration from https://lexica.art/ \n",
        "\n",
        "#@markdown ---\n",
        "start_prompt = \"A huge church, from the inside\" #@param {type:\"string\"}\n",
        "#@markdown > Optional. The opening frame, probably related to the first few words or scenery. Leave empty to ignore.\n",
        "\n",
        "#@markdown ---\n",
        "end_prompt = \"graceful death\" #@param {type:\"string\"}\n",
        "#@markdown > Optional. The last frame, probably related to the last few words or scenery. Use the same start_prompt for a boomerang effect. Leave empty to ignore.\n",
        "\n",
        "#@markdown ## 1.4 Image generation\n",
        "seed = 4000 #@param {type:\"number\"}\n",
        "#@markdown > The seed affects the style of the generated images. Tinker around to find the seed that works best to your creativity."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "cellView": "form",
        "id": "HC-s9cHPWg83",
        "outputId": "621d1c92-20ee-4067-ee52-9f2ae129a960"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/bin/bash: huggingface-cli: command not found\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown ------\n",
        "#@markdown ## ✋ Now run all the cells below. This should take ~8 minutes, depending on the duration of the transcribed video and your local machine setup.  \n",
        "#@markdown </br> Make sure there are no errors, even though this should all run smoothly.</br>\n",
        "\n",
        "#@markdown ------"
      ],
      "metadata": {
        "cellView": "form",
        "id": "Wn4Ae-4fZUdy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a2XP-0_LFIcB",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@markdown # 2. Setup Workspace"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown ## 2.1. Installing pip dependecies\n",
        "%pip install --quiet --upgrade diffusers transformers ffmpeg accelerate"
      ],
      "metadata": {
        "cellView": "form",
        "id": "R9ypQPe0klFp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "69ZI2tgLFAw4",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@markdown ## 2. FILM setup\n",
        "#@markdown ### 2.2. Google Drive setup\n",
        "\n",
        "from pathlib import Path\n",
        "import os\n",
        "\n",
        "drive_mounted = False\n",
        "gdrive_fpath = '.'\n",
        "local_path = '/content/'\n",
        "\n",
        "####################\n",
        "\n",
        "#@markdown Mounting your google drive is optional.\n",
        "#@markdown If you mount your drive, code and models will be downloaded to it.\n",
        "#@markdown This should reduce setup time after your first run.\n",
        "\n",
        "###################\n",
        "\n",
        "# Optionally Mount GDrive\n",
        "\n",
        "mount_gdrive = False # @param{type:\"boolean\"}\n",
        "if mount_gdrive and not drive_mounted:\n",
        "    from google.colab import drive\n",
        "\n",
        "    gdrive_mountpoint = '/content/drive/' #@param{type:\"string\"}\n",
        "    gdrive_subdirectory = 'MyDrive/interpolation' #@param{type:\"string\"}\n",
        "    gdrive_fpath = str(Path(gdrive_mountpoint) / gdrive_subdirectory)\n",
        "    try:\n",
        "        drive.mount(gdrive_mountpoint, force_remount = True)\n",
        "        !mkdir -p {gdrive_fpath}\n",
        "        %cd {gdrive_fpath}\n",
        "        local_path = gdrive_fpath\n",
        "        drive_mounted = True\n",
        "    except OSError:\n",
        "        print(\n",
        "            \"\\n\\n-----[PYTTI-TOOLS]-------\\n\\n\"\n",
        "            \"If you received a scary OSError and your drive\"\n",
        "            \" was already mounted, ignore it.\"\n",
        "            \"\\n\\n-----[PYTTI-TOOLS]-------\\n\\n\"\n",
        "            )\n",
        "        raise\n",
        "\n",
        "\n",
        "\n",
        "if not Path('./frame-interpolation').exists():\n",
        "    !git clone https://github.com/pytti-tools/frame-interpolation\n",
        "\n",
        "try:\n",
        "    import frame_interpolation\n",
        "except ModuleNotFoundError:\n",
        "    %pip install -r ./frame-interpolation/requirements_colab.txt\n",
        "    %pip install ./frame-interpolation\n",
        "\n",
        "#url = \"https://drive.google.com/drive/folders/1GhVNBPq20X7eaMsesydQ774CgGcDGkc6?usp=sharing\"\n",
        "share_id = \"1GhVNBPq20X7eaMsesydQ774CgGcDGkc6\" # Google FILM files\n",
        "\n",
        "if not (Path(local_path) / 'saved_model').exists():\n",
        "    !pip install --upgrade gdown\n",
        "    !gdown --folder {share_id}\n",
        "\n",
        "# create default frame\n",
        "!mkdir -p frames"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lBgvRcNAF5e9",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@markdown ### 2.2. Google FILM configuration\n",
        "\n",
        "#@markdown Specify the local directory containing your video frames with the `frames_dir` parameter.\n",
        "\n",
        "frames_dir = \"frames\" #@param{'type':'string'}\n",
        "\n",
        "#@markdown A single pass of the interpolation procedure adds a frame between each contiguous pair of frames in `frames_dir`.\n",
        "\n",
        "#@markdown If you start with $n$ frames in `frames_dir` and set `recursive_interpolation_passes` to $k$, your total number of frames\n",
        "#@markdown after interpolation will be: \n",
        "#@markdown $$2^k (n-1) -1$$\n",
        "\n",
        "import math\n",
        "film_smoothing_frames = 8 #@param{'type':'integer'}\n",
        "recursive_interpolation_passes = int(math.log2(8))\n",
        "\n",
        "#@markdown ---"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BZX7GAH_o2f_",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@markdown ## 3. OpenAI Whisper setup\n",
        "#@markdown ### 3.1. Installing dependecies\n",
        "#@markdown Whisper will be used to turn YouTube videos into transcribed prompts.\n",
        "\n",
        "#@markdown This cell will take a little while to download several libraries, including Whisper.\n",
        "\n",
        "! pip install git+https://github.com/openai/whisper.git\n",
        "! pip install pytube\n",
        "\n",
        "import sys\n",
        "import warnings\n",
        "import whisper\n",
        "from pathlib import Path\n",
        "import pytube\n",
        "import subprocess\n",
        "import torch\n",
        "import shutil\n",
        "import numpy as np\n",
        "import ffmpeg\n",
        "from IPython.display import display, Markdown, YouTubeVideo\n",
        "\n",
        "device = torch.device('cuda:0')\n",
        "print('Using device:', device, file=sys.stderr)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4G6hlu01qs76",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@markdown ### 3.2. Whisper Model selection\n",
        "\n",
        "Model = 'small.en' #@param ['tiny.en', 'tiny', 'base.en', 'base', 'small.en', 'small', 'medium.en', 'medium', 'large']\n",
        "\n",
        "whisper_model = whisper.load_model(Model)\n",
        "\n",
        "if Model in whisper.available_models():\n",
        "    display(Markdown(\n",
        "        f\"**{Model} model is selected.**\"\n",
        "    ))\n",
        "else:\n",
        "    display(Markdown(\n",
        "        f\"**{Model} model is no longer available.**<br /> Please select one of the following:<br /> - {'<br /> - '.join(whisper.available_models())}\"\n",
        "    ))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown ---"
      ],
      "metadata": {
        "cellView": "form",
        "id": "hqCu6KAA9Zga"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown # Let the magic begin 👇"
      ],
      "metadata": {
        "cellView": "form",
        "id": "LkdZH9tJpQMC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "20P1qLpaq6xC",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@markdown ## 1. Download YouTube video and save as audio\n",
        "\n",
        "video_yt = pytube.YouTube(URL)\n",
        "\n",
        "try:\n",
        "    video_yt.check_availability()\n",
        "    display(\n",
        "        YouTubeVideo(video_yt.video_id)\n",
        "    )\n",
        "except pytube.exceptions.VideoUnavailable:\n",
        "    display(\n",
        "        Markdown(f\"**{URL} isn't available.**\"),\n",
        "    )\n",
        "    raise(RuntimeError(f\"{URL} isn't available.\"))\n",
        "    \n",
        "import datetime\n",
        "file_name = f'{video_yt.video_id}_{str(datetime.datetime.now())}.wav'\n",
        "video_path_local = f'{Path(\".\")}//{file_name}'\n",
        "video_yt.streams.get_by_itag(140).download('.', file_name)\n",
        "\n",
        "output_trimed = video_path_local.split(\".\")[0]+\"_trimmed.wav\"\n",
        "audio_trimmed = ffmpeg.input(str(video_path_local), ss=start_time, t=duration)\n",
        "audio_trimmed_converted = ffmpeg.output(audio_trimmed, output_trimed)\n",
        "ffmpeg.run(audio_trimmed_converted, overwrite_output=True)\n",
        "video_path_local = output_trimed"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown ### Transcribe video into prompts\n",
        "#@markdown This can take a while and very based on the length of the video and the number of parameters of the model selected above.\n",
        "\n",
        "\n",
        "#@markdown --\n",
        "#@markdown #### Configure Whisper (optional)\n",
        "#@markdown ---\n",
        "language = \"English\" #@param ['Auto detection', 'Afrikaans', 'Albanian', 'Amharic', 'Arabic', 'Armenian', 'Assamese', 'Azerbaijani', 'Bashkir', 'Basque', 'Belarusian', 'Bengali', 'Bosnian', 'Breton', 'Bulgarian', 'Burmese', 'Castilian', 'Catalan', 'Chinese', 'Croatian', 'Czech', 'Danish', 'Dutch', 'English', 'Estonian', 'Faroese', 'Finnish', 'Flemish', 'French', 'Galician', 'Georgian', 'German', 'Greek', 'Gujarati', 'Haitian', 'Haitian Creole', 'Hausa', 'Hawaiian', 'Hebrew', 'Hindi', 'Hungarian', 'Icelandic', 'Indonesian', 'Italian', 'Japanese', 'Javanese', 'Kannada', 'Kazakh', 'Khmer', 'Korean', 'Lao', 'Latin', 'Latvian', 'Letzeburgesch', 'Lingala', 'Lithuanian', 'Luxembourgish', 'Macedonian', 'Malagasy', 'Malay', 'Malayalam', 'Maltese', 'Maori', 'Marathi', 'Moldavian', 'Moldovan', 'Mongolian', 'Myanmar', 'Nepali', 'Norwegian', 'Nynorsk', 'Occitan', 'Panjabi', 'Pashto', 'Persian', 'Polish', 'Portuguese', 'Punjabi', 'Pushto', 'Romanian', 'Russian', 'Sanskrit', 'Serbian', 'Shona', 'Sindhi', 'Sinhala', 'Sinhalese', 'Slovak', 'Slovenian', 'Somali', 'Spanish', 'Sundanese', 'Swahili', 'Swedish', 'Tagalog', 'Tajik', 'Tamil', 'Tatar', 'Telugu', 'Thai', 'Tibetan', 'Turkish', 'Turkmen', 'Ukrainian', 'Urdu', 'Uzbek', 'Valencian', 'Vietnamese', 'Welsh', 'Yiddish', 'Yoruba']\n",
        "#@markdown > Language spoken in the audio, use `Auto detection` to let Whisper detect the language.\n",
        "\n",
        "#@markdown ---\n",
        "suppress_tokens = \"-1\" #@param {type:\"string\"}\n",
        "#@markdown > Comma-separated list of token ids to suppress during sampling; '-1' will suppress most special characters except common punctuations.\n",
        "\n",
        "#@markdown ---\n",
        "initial_prompt = \"\" #@param {type:\"string\"}\n",
        "#@markdown > Optional text to provide as a prompt for the first window.\n",
        "\n",
        "#@markdown ---\n",
        "fp16 = True #@param {type:\"boolean\"}\n",
        "#@markdown > whether to perform inference in fp16.\n",
        "\n",
        "#@markdown ---\n",
        "no_speech_threshold = 0.6 #@param {type:\"slider\", min:-0.0, max:1, step:0.05}\n",
        "#@markdown > If the probability of the <|nospeech|> token is higher than this value AND the decoding has failed due to `logprob_threshold`, consider the segment as silence.\n",
        "\n",
        "#@markdown ---\n",
        "verbose = 'Live transcription' #@param ['Live transcription', 'Progress bar', 'None']\n",
        "\n",
        "verbose_lut = {\n",
        "    'Live transcription': True,\n",
        "    'Progress bar': False,\n",
        "    'None': None\n",
        "}\n",
        "\n",
        "temperature = 0.2\n",
        "temperature_increment_on_fallback = 0.2\n",
        "best_of = 5\n",
        "condition_on_previous_text=True\n",
        "\n",
        "args = dict(\n",
        "    language = (None if language == \"Auto detection\" else language),\n",
        "    verbose = verbose_lut[verbose],\n",
        "    temperature_increment_on_fallback = temperature_increment_on_fallback,\n",
        "    best_of = best_of,\n",
        "    suppress_tokens=suppress_tokens,\n",
        "    initial_prompt=(None if not initial_prompt else initial_prompt),\n",
        "    condition_on_previous_text=condition_on_previous_text,\n",
        "    fp16=fp16,\n",
        "    no_speech_threshold=no_speech_threshold\n",
        ")\n",
        "\n",
        "temperature_increment_on_fallback = args.pop(\"temperature_increment_on_fallback\")\n",
        "if temperature_increment_on_fallback is not None:\n",
        "    temperature = tuple(np.arange(temperature, 1.0 + 1e-6, temperature_increment_on_fallback))\n",
        "else:\n",
        "    temperature = [temperature]\n",
        "\n",
        "if Model.endswith(\".en\") and args[\"language\"] not in {\"en\", \"English\"}:\n",
        "    warnings.warn(f\"{Model} is an English-only model but receipted '{args['language']}'; using English instead.\")\n",
        "    args[\"language\"] = \"en\"\n",
        "\n",
        "print(\"Extracted text:\")\n",
        "video_transcription = whisper.transcribe(\n",
        "    whisper_model,\n",
        "    str(video_path_local),\n",
        "    temperature=temperature,\n",
        "    **args,\n",
        ")\n",
        "\n",
        "audio_length = float(ffmpeg.probe(f\"{video_path_local}\")[\"format\"][\"duration\"])"
      ],
      "metadata": {
        "id": "ZstcK63TxNlH",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qBQM8BYtDcUX",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@markdown ## Align prompsts with video timing\n",
        "\n",
        "prompts = [] \n",
        "for segment in video_transcription['segments']:\n",
        "  segment_text = segment['text'].strip()\n",
        "  if segment_text[0] == \"[\" and segment_text[-1] == \"]\":\n",
        "      continue\n",
        "  segment_start = segment['start']\n",
        "  segment_end = min(round(audio_length), segment['end'])\n",
        "  segment_midpoint = round(segment_start + (segment_end - segment_start)/2)\n",
        "\n",
        "  prompts.append(\n",
        "      {\n",
        "          \"prompt\": segment_text,\n",
        "          \"ts\": segment_midpoint\n",
        "      }\n",
        "  )\n",
        "\n",
        "if start_prompt:\n",
        "  prompts.insert(0, \n",
        "      {  \n",
        "          \"prompt\": start_prompt,\n",
        "          \"ts\": 0\n",
        "  })\n",
        "  \n",
        "if end_prompt:\n",
        "  prompts.append( \n",
        "      {  \n",
        "          \"prompt\": end_prompt,\n",
        "          \"ts\": round(audio_length)\n",
        "  })\n",
        "\n",
        "\n",
        "  for prompt in prompts:\n",
        "    if prompt['prompt'][:-1] == '.':\n",
        "      prompt['prompt'] += \" Image style: \" + prompt_suffix\n",
        "    else:\n",
        "      prompt['prompt'] += \". Image style: \" + prompt_suffix\n",
        "\n",
        "prompts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Rm15L_XwOErn",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@markdown # Generate images with Stable Diffusion\n",
        "#@markdown ## Download Stable Diffusion\n",
        "\n",
        "#@markdown **IMPORTANT** in case of a 403 error - approve Runway's ToS on https://huggingface.co/runwayml/stable-diffusion-v1-5\n",
        "\n",
        "from stable_diffusion_animation_pipeline import StableDiffusionAnimationPipeline\n",
        "from generation_utils import make_scheduler\n",
        "\n",
        "# Load the animation pipeline\n",
        "pipe = StableDiffusionAnimationPipeline.from_pretrained(\n",
        "    \"runwayml/stable-diffusion-v1-5\",\n",
        "    scheduler=make_scheduler(100),  # timesteps is arbitrary at this point\n",
        "    revision=\"fp16\",\n",
        "    torch_dtype=torch.float16,\n",
        ").to(\"cuda\")  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jZxxrI4dXmzm",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@markdown ## Stable Diffusion configuration\n",
        "\n",
        "guidance_scale = 7.5 #@param {type:\"number\"}\n",
        "num_inference_steps = 35 #@param {type:\"number\"}\n",
        "prompt_strength = 0.65 #@param {type:\"slider\", min:0, max: 1, step:0.01}\n",
        "fps= 24 #@param{'type':'number'}\n",
        "height = 512 #@param {type:\"number\"}\n",
        "width = 512 #@param {type:\"number\"}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7BIjSFvyOHIl",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@markdown ## Generate frames based on prompts\n",
        "\n",
        "from generation_utils import save_pil_image, slerp\n",
        "\n",
        "with torch.autocast(\"cuda\"), torch.inference_mode():\n",
        "  for i, prompt in enumerate(prompts[:-1]):\n",
        "      start_prompt = prompt\n",
        "      end_prompt = prompts[i+1]\n",
        "\n",
        "      # NOTE: To smooth out edge, the last \"n\" frames MUST be interpolated by FILM\n",
        "      # This is because edges between two segments are denoised from different midpoints\n",
        "      # and can have a jagged edge jumping between the start & end.\n",
        "      latents_start_frame_num = int(start_prompt[\"ts\"]*fps)\n",
        "      # latents_end_frame_num = int(end_prompt[\"ts\"]*fps) - edge_smoothing_frames\n",
        "      latents_end_frame_num = int(end_prompt[\"ts\"]*fps)\n",
        "\n",
        "      # Get frame difference to generate\n",
        "      frames_needed = latents_end_frame_num - latents_start_frame_num\n",
        "      # Get number of intermediate frames to be diffused based on smoothing frames\n",
        "      num_intermediate_frames = int(frames_needed/film_smoothing_frames) - 1\n",
        "\n",
        "      batch_size = 1\n",
        "\n",
        "      print(f\"Latent edge frames: {latents_start_frame_num} - {latents_end_frame_num}\")\n",
        "      print(f\"Intermediate frames needed: {num_intermediate_frames}\")\n",
        "\n",
        "      # Get start & end embeddings for prompts\n",
        "      do_classifier_free_guidance = guidance_scale > 1.0\n",
        "      text_embeddings_start = pipe.embed_text(\n",
        "        start_prompt['prompt'], do_classifier_free_guidance, batch_size\n",
        "      )\n",
        "      text_embeddings_end = pipe.embed_text(\n",
        "        end_prompt['prompt'], do_classifier_free_guidance, batch_size\n",
        "      )\n",
        "      print(text_embeddings_start.shape)\n",
        "\n",
        "      # Initialize with current start embedding as current\n",
        "      text_embeddings_current = text_embeddings_start\n",
        "\n",
        "      # Generate all intermediate frames and write all images to disk\n",
        "      print(f\"Generating {num_intermediate_frames} intermediate frames\")\n",
        "      for i in range(num_intermediate_frames + 1):\n",
        "        # For each prompt pair, create images & intermediate frames\n",
        "        generator = torch.Generator(\"cuda\").manual_seed(seed)\n",
        "\n",
        "        # Generate initial latents to start to generate animation frames from\n",
        "        initial_scheduler = pipe.scheduler = make_scheduler(\n",
        "            num_inference_steps\n",
        "        )\n",
        "\n",
        "        num_initial_steps = int(num_inference_steps * (1 - prompt_strength))\n",
        "\n",
        "        print(f\"Generating initial latents for {num_initial_steps} steps\")\n",
        "        initial_latents = torch.randn(\n",
        "            (batch_size, pipe.unet.in_channels, height // 8, width // 8),\n",
        "            generator=generator,\n",
        "            device=\"cuda\",\n",
        "        )\n",
        "\n",
        "        step_fraction = i / (num_intermediate_frames + 1)\n",
        "\n",
        "        # Get next text embedding point \n",
        "        text_embeddings_next = slerp(\n",
        "                step_fraction,\n",
        "                text_embeddings_start,\n",
        "                text_embeddings_end,\n",
        "        )\n",
        "        # Get midpoint between current and next\n",
        "        text_embeddings_mid = slerp(0.5, text_embeddings_current, text_embeddings_next)\n",
        "\n",
        "        # Get that midpoint's latents\n",
        "        latents_mid = pipe.denoise(\n",
        "            latents=initial_latents,\n",
        "            text_embeddings=text_embeddings_mid,\n",
        "            t_start=1,\n",
        "            t_end=num_initial_steps,\n",
        "            guidance_scale=guidance_scale,\n",
        "        )    \n",
        "        frame_number = int(latents_start_frame_num + (step_fraction * frames_needed))\n",
        "        print(f\"Step fraction: {step_fraction}. Frame number: {frame_number}\")\n",
        "\n",
        "        # re-initialize scheduler\n",
        "        pipe.scheduler = make_scheduler(num_inference_steps, initial_scheduler)\n",
        "\n",
        "        latents = pipe.denoise(\n",
        "            latents=latents_mid,\n",
        "            text_embeddings=text_embeddings_current,\n",
        "            t_start=num_initial_steps,\n",
        "            t_end=None,\n",
        "            guidance_scale=guidance_scale,\n",
        "        )\n",
        "\n",
        "        # Save all anchor latents to disk\n",
        "        image = pipe.latents_to_image(latents)\n",
        "        save_pil_image(\n",
        "            pipe.numpy_to_pil(image)[0], path=f\"{frames_dir}/{frame_number}\".zfill(5) + \".png\"\n",
        "        )      \n",
        "\n",
        "        text_embeddings_current = text_embeddings_next"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# clean GPU memory\n",
        "del pipe\n",
        "torch.cuda.empty_cache()"
      ],
      "metadata": {
        "id": "HoIbrY2p9kCe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown # Turn images into a video using Google FILM\n",
        "\n",
        "!python -m frame_interpolation.eval.interpolator_cli \\\n",
        "      --model_path ./saved_model \\\n",
        "      --pattern {frames_dir} \\\n",
        "      --fps {fps} \\\n",
        "      --times_to_interpolate {recursive_interpolation_passes} \\\n",
        "      --output_video"
      ],
      "metadata": {
        "id": "H1wKFYvSjnKj",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YuWuapTxu2FH",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@markdown # Combine generated video with the original audio\n",
        "!rm generated_video.mp4\n",
        "output_interpolated = f'{frames_dir}/interpolated.mp4'\n",
        "!ffmpeg -i {output_interpolated} -i {video_path_local} -c:v copy -c:a aac generated_video.mp4"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown # Your video is ready - open the files menu to your left and download `generated_video.mp4`"
      ],
      "metadata": {
        "cellView": "form",
        "id": "fTcEWNh069qk"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}