demo-pld.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "80e94e4e-8ca0-462a-9f41-95a2f41a944c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
    "device = torch.device('cuda:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b2e27a50-63e6-45c1-88c2-e43d2fa94bc5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:09<00:00,  4.78s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MistralForCausalLM(\n",
       "  (model): MistralModel(\n",
       "    (embed_tokens): Embedding(32000, 4096)\n",
       "    (layers): ModuleList(\n",
       "      (0-31): 32 x MistralDecoderLayer(\n",
       "        (self_attn): MistralAttention(\n",
       "          (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "          (k_proj): Linear(in_features=4096, out_features=1024, bias=False)\n",
       "          (v_proj): Linear(in_features=4096, out_features=1024, bias=False)\n",
       "          (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "          (rotary_emb): MistralRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): MistralMLP(\n",
       "          (gate_proj): Linear(in_features=4096, out_features=14336, bias=False)\n",
       "          (up_proj): Linear(in_features=4096, out_features=14336, bias=False)\n",
       "          (down_proj): Linear(in_features=14336, out_features=4096, bias=False)\n",
       "          (act_fn): SiLUActivation()\n",
       "        )\n",
       "        (input_layernorm): MistralRMSNorm()\n",
       "        (post_attention_layernorm): MistralRMSNorm()\n",
       "      )\n",
       "    )\n",
       "    (norm): MistralRMSNorm()\n",
       "  )\n",
       "  (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_name = \"mistralai/Mistral-7B-Instruct-v0.1\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, torch_dtype=torch.float16)\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "640f2d62-7976-4d26-beb1-08aa01043736",
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "import inspect\n",
    "import warnings\n",
    "from dataclasses import dataclass\n",
    "from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union\n",
    "\n",
    "import torch\n",
    "import torch.distributed as dist\n",
    "from torch import nn\n",
    "\n",
    "from transformers.integrations.deepspeed import is_deepspeed_zero3_enabled\n",
    "from transformers.modeling_outputs import CausalLMOutputWithPast, Seq2SeqLMOutput\n",
    "from transformers.models.auto import (\n",
    "    MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING,\n",
    "    MODEL_FOR_CAUSAL_LM_MAPPING,\n",
    "    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,\n",
    "    MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,\n",
    "    MODEL_FOR_VISION_2_SEQ_MAPPING,\n",
    ")\n",
    "from transformers.utils import ExplicitEnum, ModelOutput, is_accelerate_available, logging\n",
    "from transformers.generation.beam_constraints import DisjunctiveConstraint, PhrasalConstraint\n",
    "from transformers.generation.beam_search import BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer\n",
    "from transformers.generation.configuration_utils import GenerationConfig\n",
    "from transformers.generation.logits_process import (\n",
    "    EncoderNoRepeatNGramLogitsProcessor,\n",
    "    EncoderRepetitionPenaltyLogitsProcessor,\n",
    "    EpsilonLogitsWarper,\n",
    "    EtaLogitsWarper,\n",
    "    ExponentialDecayLengthPenalty,\n",
    "    ForcedBOSTokenLogitsProcessor,\n",
    "    ForcedEOSTokenLogitsProcessor,\n",
    "    ForceTokensLogitsProcessor,\n",
    "    HammingDiversityLogitsProcessor,\n",
    "    InfNanRemoveLogitsProcessor,\n",
    "    LogitNormalization,\n",
    "    LogitsProcessorList,\n",
    "    MinLengthLogitsProcessor,\n",
    "    MinNewTokensLengthLogitsProcessor,\n",
    "    NoBadWordsLogitsProcessor,\n",
    "    NoRepeatNGramLogitsProcessor,\n",
    "    PrefixConstrainedLogitsProcessor,\n",
    "    RepetitionPenaltyLogitsProcessor,\n",
    "    SequenceBiasLogitsProcessor,\n",
    "    SuppressTokensAtBeginLogitsProcessor,\n",
    "    SuppressTokensLogitsProcessor,\n",
    "    TemperatureLogitsWarper,\n",
    "    TopKLogitsWarper,\n",
    "    TopPLogitsWarper,\n",
    "    TypicalLogitsWarper,\n",
    "    UnbatchedClassifierFreeGuidanceLogitsProcessor,\n",
    ")\n",
    "from transformers.generation.stopping_criteria import (\n",
    "    MaxLengthCriteria,\n",
    "    MaxTimeCriteria,\n",
    "    StoppingCriteria,\n",
    "    StoppingCriteriaList,\n",
    "    validate_stopping_criteria,\n",
    ")\n",
    "\n",
    "from transformers.generation.utils import _crop_past_key_values\n",
    "\n",
    "@dataclass\n",
    "class GreedySearchDecoderOnlyOutput(ModelOutput):\n",
    "    \"\"\"\n",
    "    Base class for outputs of decoder-only generation models using greedy search.\n",
    "\n",
    "\n",
    "    Args:\n",
    "        sequences (`torch.LongTensor` of shape `(batch_size, sequence_length)`):\n",
    "            The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter\n",
    "            if all batches finished early due to the `eos_token_id`.\n",
    "        scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):\n",
    "            Processed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)\n",
    "            at each generation step. Tuple of `torch.FloatTensor` with up to `max_new_tokens` elements (one element for\n",
    "            each generated token), with each tensor of shape `(batch_size, config.vocab_size)`.\n",
    "        attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):\n",
    "            Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of\n",
    "            `torch.FloatTensor` of shape `(batch_size, num_heads, generated_length, sequence_length)`.\n",
    "        hidden_states (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):\n",
    "            Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of\n",
    "            `torch.FloatTensor` of shape `(batch_size, generated_length, hidden_size)`.\n",
    "    \"\"\"\n",
    "\n",
    "    sequences: torch.LongTensor = None\n",
    "    scores: Optional[Tuple[torch.FloatTensor]] = None\n",
    "    attentions: Optional[Tuple[Tuple[torch.FloatTensor]]] = None\n",
    "    hidden_states: Optional[Tuple[Tuple[torch.FloatTensor]]] = None\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d5466bcc-865b-4333-9cae-4d3fa4afd693",
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def find_candidate_pred_tokens(input_ids, max_ngram_size=3, num_pred_tokens=10):\n",
    "    input_length = input_ids.size(1)\n",
    "\n",
    "    # Ensure max_ngram_size and num_pred_tokens are valid\n",
    "    if max_ngram_size <= 0 or num_pred_tokens <= 0 or max_ngram_size > input_length:\n",
    "        raise ValueError(\"Invalid max_ngram_size or num_pred_tokens\")\n",
    "\n",
    "    for ngram_size in range(max_ngram_size, 0, -1):\n",
    "        # Extract the last n tokens as our search ngram\n",
    "        ngram = input_ids[0, -ngram_size:].tolist()\n",
    "\n",
    "        # Create sliding windows of size ngram_size\n",
    "        windows = input_ids.unfold(dimension=1, size=ngram_size, step=1)\n",
    "\n",
    "        # Convert ngram to a tensor for comparison\n",
    "        ngram_tensor = torch.tensor(ngram, device=input_ids.device).unsqueeze(0)\n",
    "\n",
    "        # Find where the windows match the ngram\n",
    "        matches = (windows == ngram_tensor).all(dim=2)\n",
    "\n",
    "        # Get the indices of matches\n",
    "        match_indices = matches.nonzero(as_tuple=True)[1]\n",
    "\n",
    "        # Iterate through match indices to find a valid continuation\n",
    "        for idx in match_indices:\n",
    "            start_idx = idx + ngram_size\n",
    "            end_idx = start_idx + num_pred_tokens\n",
    "            # Ensure we don't go beyond the length of input_ids and avoid self-match\n",
    "            if end_idx <= input_length and start_idx < input_length - ngram_size:\n",
    "                return input_ids[0, start_idx:end_idx]\n",
    "\n",
    "    # If no match is found, return an empty tensor\n",
    "    return torch.tensor([], dtype=torch.long, device=input_ids.device)\n",
    "\n",
    "COLORS = [\"\\x1b[31m\", \"\\x1b[32m\", \"\\x1b[34m\", \"\\x1b[35m\"]  # Red, Green, Blue, Magenta\n",
    "UNDERLINE = \"\\x1b[4m\"\n",
    "RESET = \"\\x1b[0m\"\n",
    "\n",
    "@torch.no_grad()\n",
    "def greedy_search_pld(\n",
    "        self,\n",
    "        input_ids: torch.LongTensor,\n",
    "        logits_processor: Optional[LogitsProcessorList] = None,\n",
    "        stopping_criteria: Optional[StoppingCriteriaList] = None,\n",
    "        max_length: Optional[int] = None,\n",
    "        pad_token_id: Optional[int] = None,\n",
    "        eos_token_id: Optional[Union[int, List[int]]] = None,\n",
    "        output_attentions: Optional[bool] = None,\n",
    "        output_hidden_states: Optional[bool] = None,\n",
    "        output_scores: Optional[bool] = None,\n",
    "        return_dict_in_generate: Optional[bool] = None,\n",
    "        synced_gpus: bool = False,\n",
    "        streamer: Optional[\"BaseStreamer\"] = None,\n",
    "        draft_matching_window_size = 3,\n",
    "        draft_num_candidate_tokens = 10,\n",
    "        print_output=True,\n",
    "        **model_kwargs,\n",
    "    ):\n",
    "\n",
    "        global tokenizer\n",
    "\n",
    "        # init values\n",
    "        stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()\n",
    "        pad_token_id = pad_token_id if pad_token_id is not None else self.generation_config.pad_token_id\n",
    "        eos_token_id = eos_token_id if eos_token_id is not None else self.generation_config.eos_token_id\n",
    "        if isinstance(eos_token_id, int):\n",
    "            eos_token_id = [eos_token_id]\n",
    "        eos_token_id_tensor = torch.tensor(eos_token_id).to(input_ids.device) if eos_token_id is not None else None\n",
    "\n",
    "        # # init attention / hidden states / scores tuples\n",
    "        scores = () if (return_dict_in_generate and output_scores) else None\n",
    "\n",
    "        max_len = stopping_criteria[0].max_length\n",
    "\n",
    "        i = 0\n",
    "        current_color_index = 0\n",
    "\n",
    "        while True:\n",
    "            i += 1\n",
    "            cur_len = input_ids.shape[-1]\n",
    "\n",
    "            candidate_pred_tokens = find_candidate_pred_tokens(input_ids, draft_matching_window_size, draft_num_candidate_tokens)\n",
    "\n",
    "            if len(candidate_pred_tokens) == 0:\n",
    "                candidate_pred_tokens = torch.tensor([100], device=input_ids.device).unsqueeze(0)\n",
    "            else:\n",
    "                candidate_pred_tokens = candidate_pred_tokens.unsqueeze(0)\n",
    "            \n",
    "            candidate_input_ids = torch.cat((input_ids, candidate_pred_tokens), dim=1)\n",
    "            \n",
    "            candidate_length = candidate_input_ids.shape[1] - input_ids.shape[1]\n",
    "\n",
    "            candidate_kwargs = copy.copy(model_kwargs)\n",
    "            candidate_kwargs = self._extend_attention_mask(candidate_kwargs, candidate_input_ids.shape[1])\n",
    "            candidate_kwargs = self._extend_token_type_ids(candidate_kwargs, candidate_input_ids.shape[1])\n",
    "\n",
    "            model_inputs = self.prepare_inputs_for_generation(candidate_input_ids, **candidate_kwargs)\n",
    "            \n",
    "            # prepare model inputs\n",
    "            # model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)\n",
    "\n",
    "            # forward pass to get next token\n",
    "            outputs = self(\n",
    "                **model_inputs,\n",
    "                return_dict=True,\n",
    "                output_attentions=output_attentions,\n",
    "                output_hidden_states=output_hidden_states,\n",
    "            )\n",
    "\n",
    "\n",
    "            new_logits = outputs.logits[:, -candidate_length - 1 :]  # excludes the input prompt if present\n",
    "            selected_tokens = new_logits.argmax(dim=-1)\n",
    "            candidate_new_tokens = candidate_input_ids[:, -candidate_length:]\n",
    "            n_matches = ((~(candidate_new_tokens == selected_tokens[:, :-1])).cumsum(dim=-1) < 1).sum()\n",
    "\n",
    "            \n",
    "            # if last_assistant_token_is_eos and n_matches == candidate_length: # todo: do this earlier somehow\n",
    "            #     n_matches -= 1\n",
    "            \n",
    "            n_matches = min(n_matches, max_len - cur_len - 1)\n",
    "\n",
    "            # print(n_matches)\n",
    "            # i+= n_matches.item()\n",
    "\n",
    "            if print_output:\n",
    "                current_text = tokenizer.decode(input_ids[0], skip_special_tokens=True)\n",
    "            \n",
    "            valid_tokens = selected_tokens[:, : n_matches + 1]\n",
    "            input_ids = torch.cat((input_ids, valid_tokens), dim=-1)\n",
    "            new_cur_len = input_ids.shape[-1]\n",
    "\n",
    "            if print_output:\n",
    "                updated_text = tokenizer.decode(input_ids[0], skip_special_tokens=True)\n",
    "                # Find and print the newly added text\n",
    "                if updated_text != current_text:\n",
    "                    new_text = updated_text[len(current_text):]\n",
    "                    if len(valid_tokens[0]) > 1:\n",
    "                        color = COLORS[current_color_index]\n",
    "                        print(f\"{color}{new_text}{RESET}\", end='')\n",
    "                        # Update color for next generation\n",
    "                        current_color_index = (current_color_index + 1) % len(COLORS)\n",
    "                    else:\n",
    "                        print(f\"{new_text}\", end='')\n",
    "\n",
    "            new_cache_size = new_cur_len - 1\n",
    "            outputs.past_key_values = _crop_past_key_values(self, outputs.past_key_values, new_cache_size)\n",
    "\n",
    "        \n",
    "            model_kwargs[\"past_key_values\"] = outputs.past_key_values\n",
    "\n",
    "            # stop if we exceed the maximum length\n",
    "\n",
    "            if (valid_tokens == eos_token_id_tensor.item()).any():\n",
    "                break\n",
    "            \n",
    "            if stopping_criteria(input_ids, scores):\n",
    "                break\n",
    "\n",
    "\n",
    "        if return_dict_in_generate:\n",
    "            return GreedySearchDecoderOnlyOutput(\n",
    "                sequences=input_ids,\n",
    "                scores=scores,\n",
    "                # attentions=decoder_attentions,\n",
    "                # hidden_states=decoder_hidden_states,\n",
    "            )\n",
    "        else:\n",
    "            return input_ids\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a9dcce41-44b2-4260-b844-8348b785d7a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.greedy_search_pld = greedy_search_pld.__get__(model, type(model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "575a21f7-2aed-47ab-9f27-7c52a3c506fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "code_text = \"\"\"import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Calculate the average\n",
    "average_throughput = np.mean(tokens_per_sec_arr)\n",
    "print(f\"Average Throughput: {average_throughput} tokens/sec\")\n",
    "\n",
    "# Plotting the histogram\n",
    "plt.hist(tokens_per_sec_arr, bins=20, color='blue', edgecolor='black', alpha=0.7)\n",
    "plt.title('Histogram of Throughput Values')\n",
    "plt.xlabel('Tokens per Second')\n",
    "plt.ylabel('Frequency')\n",
    "plt.axvline(average_throughput, color='red', linestyle='dashed', linewidth=1)\n",
    "plt.text(average_throughput*0.9, max(plt.ylim())*0.9, f'Average: {average_throughput:.2f}', color = 'red')\n",
    "plt.show()\n",
    "\"\"\"\n",
    "question = \"Can you please change x axis to start from 0\"\n",
    "prompt = \"[INST] Code:```python\\n{code_text}``` \\n\\n Question: {question} \\n\\n Modified code:[/INST]\".format(code_text=code_text, question=question)\n",
    "\n",
    "inputs = tokenizer(prompt, return_tensors=\"pt\")\n",
    "\n",
    "# Move all tensor values in the inputs to GPU\n",
    "for key in inputs:\n",
    "    inputs[key] = inputs[key].to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b9527e26-56e7-48ff-bb11-724f88ea0fa6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Sure, here's the modified code\u001b[31m:\n",
      "\u001b[0m``\u001b[32m`python\u001b[0m\u001b[34m\n",
      "import numpy as np\n",
      "import matplotlib.\u001b[0m\u001b[35mpyplot as plt\n",
      "\n",
      "# Calculate the average\u001b[0m\u001b[31m\n",
      "average_throughput = np.mean(\u001b[0m\u001b[32mtokens_per_sec_arr)\n",
      "print(\u001b[0m\u001b[34mf\"Average Throughput: {average_\u001b[0m\u001b[35mthroughput}\u001b[0m\u001b[31m tokens/sec\")\n",
      "\n",
      "# Plotting the hist\u001b[0m\u001b[32mogram\n",
      "plt.hist(tokens_per_\u001b[0m\u001b[34msec_arr,\u001b[0m\u001b[35m bins=20, color='blue', edge\u001b[0m\u001b[31mcolor='black', alpha=0.7)\n",
      "\u001b[0m\u001b[32mplt.title('Histogram of Throughput Val\u001b[0m\u001b[34mues')\n",
      "plt.xlabel('Tokens per\u001b[0m\u001b[35m Second')\n",
      "plt.ylabel('Frequency\u001b[0m\u001b[31m')\n",
      "plt.x\u001b[0mlim(0\u001b[32m, max\u001b[0m\u001b[34m(tokens\u001b[0m\u001b[35m_per_sec_arr))\u001b[0m\n",
      "pl\u001b[31mt.ax\u001b[0m\u001b[32mvline(average_throughput, color='\u001b[0mred\u001b[34m', linestyle='dashed', linewidth=1\u001b[0m\u001b[35m)\n",
      "plt.text(average_through\u001b[0m\u001b[31mput*\u001b[0m\u001b[32m0.9, max(plt.ylim\u001b[0m\u001b[34m())*0.9, f'Average:\u001b[0m\u001b[35m {average_throughput:.2f}',\u001b[0m\u001b[31m color = 'red')\n",
      "plt.show()\u001b[0m\u001b[32m\n",
      "```\n",
      "\u001b[0mIn this modified\u001b[34m code,\u001b[0m we added `pl\u001b[35mt.x\u001b[0mlim\u001b[31m(0, max(tokens_per_sec_\u001b[0m\u001b[32marr))\u001b[0m` to set the x-axis limits to\u001b[34m start from 0.\u001b[0m\n",
      "\n",
      "Total time: 2.3677501678466797 seconds\n",
      "Tokens per second: 127.54724045681309 tokens/sec\n",
      "Total tokens generated: 302\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from transformers import StoppingCriteriaList, MaxLengthCriteria\n",
    "\n",
    "# Define the variable for max_new_tokens\n",
    "max_new_tokens = 500\n",
    "use_new_generate = True\n",
    "\n",
    "# Start timing\n",
    "start_time = time.time()\n",
    "\n",
    "# Generate the output\n",
    "\n",
    "if not use_new_generate:\n",
    "    out = model.generate(inputs=inputs.input_ids, max_new_tokens=max_new_tokens, use_cache=True, pad_token_id=0,\n",
    "                         do_sample=False,\n",
    "                         return_dict_in_generate=True)\n",
    "else:\n",
    "    out = model.greedy_search_pld(inputs.input_ids, \n",
    "                              attention_mask = inputs.attention_mask,\n",
    "                              stopping_criteria=StoppingCriteriaList([MaxLengthCriteria(max_length=len(inputs.input_ids[0]) + max_new_tokens)]),\n",
    "                              draft_matching_window_size = 3,\n",
    "                              draft_num_candidate_tokens = 10,\n",
    "                              use_cache=True, \n",
    "                              pad_token_id=0,\n",
    "                              eos_token_id=2,\n",
    "                              return_dict_in_generate=True)\n",
    "\n",
    "end_time = time.time()\n",
    "\n",
    "out_text = tokenizer.batch_decode(out.sequences, skip_special_tokens=True)[0]\n",
    "\n",
    "\n",
    "num_tokens_generated = len(out.sequences[0]) - len(inputs['input_ids'][0])\n",
    "\n",
    "total_time = end_time - start_time\n",
    "tokens_per_sec = num_tokens_generated / total_time\n",
    "\n",
    "print(f\"\\n\\nTotal time: {total_time} seconds\")\n",
    "print(f\"Tokens per second: {tokens_per_sec} tokens/sec\")\n",
    "print(f\"Total tokens generated: {num_tokens_generated}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51bc4f6f-e6db-47df-a537-8534a5a4430f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}