main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the parameters\n",
    "##############################\n",
    "player_name = 'V'\n",
    "game_name = 'Cyberpunk_2077'\n",
    "interact_key = 't'\n",
    "facial_animation_switch = True\n",
    "##############################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "from functions.grabscreen import grab_screen\n",
    "import time\n",
    "from functions.speech_to_text import speech_to_text\n",
    "import speech_recognition as sr\n",
    "import os\n",
    "from functions.main import main\n",
    "import threading\n",
    "from pydub import AudioSegment\n",
    "from pydub.playback import play\n",
    "\n",
    "# Get Character List\n",
    "characters = [entry.name for entry in os.scandir(os.path.join(game_name, 'characters')) if entry.is_dir() and entry.name != 'default']\n",
    "\n",
    "# Define the region of the screen\n",
    "left = 0\n",
    "top = 0\n",
    "width = 1920\n",
    "height = 1080\n",
    "\n",
    "# Create a window to display the captured screen\n",
    "cv2.namedWindow(\"Screen Capture\", cv2.WINDOW_NORMAL)\n",
    "\n",
    "# Adjust the window size to match the captured screen size\n",
    "cv2.resizeWindow(\"Screen Capture\", width, height)\n",
    "\n",
    "font = cv2.FONT_HERSHEY_TRIPLEX\n",
    "font_scale = 0.7\n",
    "font_color = (192, 192, 192)  # Light grey color\n",
    "line_thickness = 1\n",
    "\n",
    "speak_display = False\n",
    "speech_recognition_start = False\n",
    "start_time = 0\n",
    "\n",
    "transcribed_text = \"NULL\"\n",
    "speech_to_text_error = \"NULL\"\n",
    "transcribed_text_display = False\n",
    "speech_to_text_error_display = False\n",
    "\n",
    "main_function_start = False\n",
    "\n",
    "# Initialise speech recognition\n",
    "r = sr.Recognizer()\n",
    "\n",
    "def play_audio():\n",
    "    # Load and play the audio\n",
    "    audio = AudioSegment.from_file('temp/audio.wav')\n",
    "    play(audio)\n",
    "\n",
    "while True:\n",
    "    screen = grab_screen(region=(left, top, left + width, top + height))\n",
    "    screen = cv2.cvtColor(screen, cv2.COLOR_BGR2RGB)\n",
    "    text_position = (screen.shape[1] - 420, 50)\n",
    "\n",
    "    if speak_display:\n",
    "        cv2.putText(screen, \"Speak\", text_position, font, font_scale, font_color, line_thickness)\n",
    "        elapsed_time = time.time() - start_time\n",
    "        if elapsed_time < 2:\n",
    "            speech_recognition_start = False\n",
    "        else:\n",
    "            speech_recognition_start = True\n",
    "\n",
    "    if transcribed_text_display:\n",
    "        # Wrap the text if it exceeds a certain length\n",
    "        text_lines = []\n",
    "        line_start = 0\n",
    "        line_end = 30  # Maximum characters per line\n",
    "\n",
    "        while line_start < len(transcribed_text):\n",
    "            text_lines.append(transcribed_text[line_start:line_end])\n",
    "            line_start = line_end\n",
    "            line_end += 30\n",
    "\n",
    "        for i, line in enumerate(text_lines):\n",
    "            y = text_position[1] + (i * 30)\n",
    "            cv2.putText(screen, line, (text_position[0], y), font, font_scale, font_color, line_thickness)\n",
    "\n",
    "        elapsed_time = time.time() - start_time\n",
    "        if elapsed_time < 3:\n",
    "            transcribed_text_display = True\n",
    "        else:\n",
    "            transcribed_text_display = False\n",
    "            main_function_start = True\n",
    "\n",
    "    if speech_to_text_error_display:\n",
    "        # Wrap the text if it exceeds a certain length\n",
    "        text_lines = []\n",
    "        line_start = 0\n",
    "        line_end = 30  # Maximum characters per line\n",
    "\n",
    "        while line_start < len(speech_to_text_error):\n",
    "            text_lines.append(speech_to_text_error[line_start:line_end])\n",
    "            line_start = line_end\n",
    "            line_end += 30\n",
    "\n",
    "        for i, line in enumerate(text_lines):\n",
    "            y = text_position[1] + (i * 30)\n",
    "            cv2.putText(screen, line, (text_position[0], y), font, font_scale, font_color, line_thickness)\n",
    "\n",
    "        elapsed_time = time.time() - start_time\n",
    "        if elapsed_time < 3:\n",
    "            speech_to_text_error_display = True\n",
    "        else:\n",
    "            speech_to_text_error_display = False\n",
    "\n",
    "    if speech_recognition_start:\n",
    "        with sr.Microphone() as source:\n",
    "            audio = r.listen(source)\n",
    "\n",
    "        transcribed_text, speech_to_text_error = speech_to_text(audio)\n",
    "        if transcribed_text != 'NULL':\n",
    "            transcribed_text_display = True\n",
    "            start_time = time.time()\n",
    "        else:\n",
    "            speech_to_text_error_display = True\n",
    "            start_time = time.time()\n",
    "\n",
    "        speak_display = False\n",
    "        speech_recognition_start = False\n",
    "\n",
    "    if main_function_start:\n",
    "        facial_animation_video_path, audio_path, coordinates  = main(screen, transcribed_text, player_name, game_name, characters, facial_animation_switch)\n",
    "        if facial_animation_video_path == \"\":\n",
    "            play_audio()\n",
    "            main_function_start = False\n",
    "            continue\n",
    "        x = coordinates[0]\n",
    "        y = coordinates[1]\n",
    "        w = coordinates[2]\n",
    "        h = coordinates[3]\n",
    "\n",
    "        # Save the Screen\n",
    "        cv2.imwrite('temp/screen.jpg', screen)\n",
    "\n",
    "        # Read the image file\n",
    "        image = cv2.imread('temp/screen.jpg')\n",
    "\n",
    "        # Read the video file\n",
    "        video = cv2.VideoCapture(facial_animation_video_path)\n",
    "\n",
    "        # Get video properties\n",
    "        fps = video.get(cv2.CAP_PROP_FPS)\n",
    "\n",
    "        # Start the audio playback in a separate thread\n",
    "        audio_thread = threading.Thread(target=play_audio)\n",
    "        audio_thread.start()\n",
    "\n",
    "        while True:\n",
    "\n",
    "            # Read a frame from the video\n",
    "            ret, frame = video.read()\n",
    "\n",
    "            if not ret:\n",
    "                main_function_start = False\n",
    "                break\n",
    "\n",
    "            # Resize the frame to match ROI dimensions\n",
    "            frame = cv2.resize(frame, (w, h))\n",
    "\n",
    "            # Replace the ROI in the image with the frame\n",
    "            image[y:y+h, x:x+w] = frame\n",
    "\n",
    "            # Display the image with the video overlay\n",
    "            cv2.imshow('Screen Capture', image)\n",
    "\n",
    "            # Delay to control the frame rate\n",
    "            delay = int(1000 / fps)  # Calculate the delay based on video frame rate\n",
    "            if cv2.waitKey(delay) == ord('q'):\n",
    "                break\n",
    "\n",
    "        # Release resources\n",
    "        video.release()\n",
    "\n",
    "        # Wait for the audio playback thread to finish\n",
    "        audio_thread.join()\n",
    "\n",
    "    cv2.imshow(\"Screen Capture\", screen)\n",
    "\n",
    "    # Wait for a key press and break the loop if 'q' is pressed\n",
    "    if cv2.waitKey(1) == ord(\"q\"):\n",
    "        break\n",
    "\n",
    "    if cv2.waitKey(1) == ord(interact_key) and not speak_display:\n",
    "        speak_display = True\n",
    "        start_time = time.time()\n",
    "\n",
    "# Release the window and resources\n",
    "cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}