diarization

machine_learning/controlnet.ipynb

@@ -0,0 +1,289 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "485400f3",
+   "metadata": {},
+   "source": [
+    "## controlnet\n",
+    "\n",
+    "* https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/controlnet.ipynb#scrollTo=wsv55Py8onJx"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "07599106-0e2a-4254-95df-c2668c226c61",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import os, glob\n",
+    "import random, math\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import torch\n",
+    "styles = pd.read_csv('stable_diffusion_artist_styles.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "c998ace4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from diffusers.utils import load_image\n",
+    "\n",
+    "def conditioning(image):\n",
+    "    import cv2\n",
+    "    from PIL import Image\n",
+    "\n",
+    "    image = np.array(image)\n",
+    "\n",
+    "    low_threshold = 50\n",
+    "    high_threshold = 200\n",
+    "\n",
+    "    image = cv2.Canny(image, low_threshold, high_threshold)\n",
+    "    image = image[:, :, None]\n",
+    "    image = np.concatenate([image, image, image], axis=2)\n",
+    "    canny_image = Image.fromarray(image)\n",
+    "    return canny_image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "f96d3a32",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/storage/local/stablediff/lib/python3.10/site-packages/safetensors/torch.py:99: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  with safe_open(filename, framework=\"pt\", device=device) as f:\n"
+     ]
+    }
+   ],
+   "source": [
+    "from diffusers import StableDiffusionControlNetPipeline, ControlNetModel\n",
+    "import torch\n",
+    "\n",
+    "controlnet = ControlNetModel.from_pretrained(\"lllyasviel/sd-controlnet-canny\", torch_dtype=torch.float16, safety_checker=None)\n",
+    "pipe = StableDiffusionControlNetPipeline.from_pretrained(\n",
+    "    \"runwayml/stable-diffusion-v1-5\", controlnet=controlnet, torch_dtype=torch.float16\n",
+    ")\n",
+    "from diffusers import UniPCMultistepScheduler\n",
+    "\n",
+    "pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)\n",
+    "pipe.enable_model_cpu_offload()\n",
+    "pipe.enable_xformers_memory_efficient_attention()\n",
+    "pipe.safety_checker = lambda images, clip_input: (images, False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "d445bca8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def controlnet_prompt(prompt, canny_img, n=1, style=None, path='.'):\n",
+    "    if style != None:\n",
+    "        prompt += ' by %s'%style\n",
+    "    #prompt += \", best quality, extremely detailed, studio photograph, cinematic\"\n",
+    "    for c in range(n):\n",
+    "        random_seed = np.random.randint(1000)\n",
+    "        generator = torch.Generator(device=\"cpu\").manual_seed(random_seed)\n",
+    "        output = pipe(\n",
+    "            prompt,\n",
+    "            canny_img,\n",
+    "            negative_prompt=\"disfigured, monochrome, lowres, bad anatomy, worst quality, low quality\" * len(prompt),\n",
+    "            generator=generator,\n",
+    "            num_inference_steps=20,\n",
+    "        )\n",
+    "        image = output.images[0]\n",
+    "        if not os.path.exists(path):\n",
+    "            os.makedirs(path)        \n",
+    "        i=1\n",
+    "        imgfile = os.path.join(path,prompt[:90]+'_%s.png' %i)\n",
+    "        while os.path.exists(imgfile):\n",
+    "            i+=1\n",
+    "            imgfile = os.path.join(path,prompt[:90]+'_%s.png' %i)\n",
+    "        image.save(imgfile,'png')           \n",
+    "    return image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "45b7d50f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#image = load_image('superman-pose2.jpg')\n",
+    "\n",
+    "canny_image = conditioning(image)\n",
+    "canny_image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "17e6d010",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "controlnet_prompt(\"child, oil impressionist\",\n",
+    "                  canny_image,n=5,style='',path='controlnet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "f7892594",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def make_montage(cond_img, names, style, path, other=''):\n",
+    "    \"\"\"montage from conditional image\"\"\"\n",
+    "\n",
+    "    canny_image = conditioning(cond_img)\n",
+    "    for name in names:\n",
+    "        controlnet_prompt(name+' '+other,cond_img,n=1,style=style,path=path)\n",
+    "    return"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d367d5f8-4d19-4d27-9a01-6d60cbcdfdd6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pres=['lyndon johnson','dwight eisenhower','harry truman','woodrow wilson','teddy roosevelt','franklin roosevelt',\n",
+    "       'john f kennedy','richard nixon','gerald ford','jimmy carter','joe biden',\n",
+    "      'donald trump','george w bush','george hw bush','bill clinton','barack obama'\n",
+    "      ]\n",
+    "actors = ['tom cruise','brad pitt','chris pine','arnold schwarzenegger',\n",
+    "          'sylvester stallone','chris hemsworth','chris pratt','henry cavill'\n",
+    "          ]\n",
+    "make_montage(canny_image, actors, 'robert bateman', 'pose', other='superhero')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "29850a61",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def make_gif(path, outfile):\n",
+    "    \"\"\"make gif from same condition image\"\"\"\n",
+    "\n",
+    "    import glob\n",
+    "    from PIL import Image\n",
+    "    files = glob.glob(path+'/*.png')\n",
+    "    #print (files)\n",
+    "    img = Image.open(files[0])    \n",
+    "    #size = img.size\n",
+    "    images=[]\n",
+    "    images = [Image.open(f) for f in files]\n",
+    "    images[0].save(outfile,\n",
+    "               save_all=True, append_images=images[1:], \n",
+    "               optimize=False, duration=400, loop=0)\n",
+    "    return\n",
+    "\n",
+    "make_gif('superman','superman.gif')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "id": "15ab901a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def tile_images(image_paths, outfile, grid=False, tile_width=300):\n",
+    "    \"\"\"Make tiled image\"\"\"\n",
+    "\n",
+    "    from PIL import Image, ImageDraw\n",
+    "    images = [Image.open(path) for path in image_paths]\n",
+    "      \n",
+    "    ratio = images[0].height / images[0].width\n",
+    "    tile_height = int( tile_width * ratio )\n",
+    "    num_rows = int(math.sqrt(len(image_paths)))\n",
+    "    # Calculate number of cols\n",
+    "    num_columns = (len(images) + num_rows - 1) // num_rows\n",
+    "\n",
+    "    tiled_width = num_columns * tile_width\n",
+    "    tiled_height = num_rows * tile_height\n",
+    "    tiled_image = Image.new(\"RGB\", (tiled_width, tiled_height))\n",
+    "\n",
+    "    for idx, image in enumerate(images):      \n",
+    "        row = idx // num_columns\n",
+    "        col = idx % num_columns\n",
+    "        x_offset = col * tile_width\n",
+    "        y_offset = row * tile_height\n",
+    "        tiled_image.paste(image.resize((tile_width, tile_height)), (x_offset, y_offset))\n",
+    "    if grid == True:\n",
+    "        draw = ImageDraw.Draw(tiled_image)\n",
+    "        # Draw borders around each tile\n",
+    "        for row in range(num_rows):\n",
+    "            for col in range(num_columns):\n",
+    "                x1 = col * tile_width\n",
+    "                y1 = row * tile_height\n",
+    "                x2 = x1 + tile_width\n",
+    "                y2 = y1 + tile_height\n",
+    "                draw.rectangle([x1, y1, x2, y2], outline=(0, 0, 0), width=3)  \n",
+    "\n",
+    "    tiled_image.save(outfile)\n",
+    "    return tiled_image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e98b903b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "files = glob.glob('mona/*.png')\n",
+    "x = random.sample(files, 3)\n",
+    "tile_images(x, 'tiled.png', grid=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bb6dc2fe",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "stablediff",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}