Problems with using JavaCV in Minecraft

[dousha0w0]

`package io.github.smartdousha.kideo.client;

import net.minecraft.client.MinecraftClient;
import org.bytedeco.ffmpeg.global.avutil;
import org.bytedeco.javacv.FFmpegFrameGrabber;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.FrameGrabber.Exception;
import org.lwjgl.glfw.GLFWErrorCallback;
import org.lwjgl.glfw.GLFWVidMode;
import org.lwjgl.opengl.GL;
import org.lwjgl.system.MemoryStack;

import javax.sound.sampled.*;
import java.io.File;
import java.nio.ByteBuffer;
import java.nio.IntBuffer;
import java.nio.ShortBuffer;
import java.util.Objects;

import static org.lwjgl.glfw.Callbacks.glfwFreeCallbacks;
import static org.lwjgl.glfw.GLFW.;
import static org.lwjgl.opengl.GL11.;
import static org.lwjgl.opengl.GL12.GL_BGR;
import static org.lwjgl.opengl.GL12.GL_CLAMP_TO_EDGE;
import static org.lwjgl.system.MemoryStack.stackPush;
import static org.lwjgl.system.MemoryUtil.NULL;

public class Video {
private final int width;
private final int height;
private long window;
private final FFmpegFrameGrabber grabber;

public Video(String path) throws Exception {
this.grabber = new FFmpegFrameGrabber(new File(path));
this.grabber.setPixelFormat(avutil.AV_PIX_FMT_RGB24);
this.grabber.start();
this.width = this.grabber.getImageWidth();
this.height = this.grabber.getImageHeight();
}

public void play() throws Exception, LineUnavailableException {
loop();
}

private void init() {
GLFWErrorCallback.createPrint(System.err).set();
window = MinecraftClient.getInstance().getWindow().getHandle();

if (!glfwInit()) {
  throw new IllegalStateException("Unable to initialize GLFW");
}

glfwDefaultWindowHints();
glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE);
glfwWindowHint(GLFW_RESIZABLE, GLFW_TRUE);

if (window == NULL) {
  throw new RuntimeException("Failed to create the GLFW window");
}

try (MemoryStack stack = stackPush()) {
  IntBuffer pWidth = stack.mallocInt(1);
  IntBuffer pHeight = stack.mallocInt(1);

  glfwGetWindowSize(window, pWidth, pHeight);

  GLFWVidMode vidmode = glfwGetVideoMode(glfwGetPrimaryMonitor());

  assert vidmode != null;
  glfwSetWindowPos(
      window, (vidmode.width() - pWidth.get(0)) / 2, (vidmode.height() - pHeight.get(0)) / 2);
}
glfwMakeContextCurrent(window);

glfwSwapInterval(1);
glfwShowWindow(window);

}

private void loop() throws LineUnavailableException {
this.init();

final AudioFormat audioFormat = new AudioFormat(grabber.getSampleRate(), 16, grabber.getAudioChannels(), true, true);
final DataLine.Info info = new DataLine.Info(SourceDataLine.class, audioFormat);
final SourceDataLine soundLine = (SourceDataLine) AudioSystem.getLine(info);
soundLine.open(audioFormat);
soundLine.start();

GL.createCapabilities();

glClearColor(0.0F, 0.0F, 0.0F, 0.0F);
glMatrixMode(GL_PROJECTION);
glLoadIdentity();
glOrtho(0, this.width, this.height, 0, 1, -1);
glMatrixMode(GL_MODELVIEW);
glEnable(GL_TEXTURE_2D);
int texture = glGenTextures();
glBindTexture(GL_TEXTURE_2D, texture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(
    GL_TEXTURE_2D,
    0,
    GL_RGB,
    this.width,
    this.height,
    0,
    GL_BGR,
    GL_UNSIGNED_BYTE,
    (ByteBuffer) null);
while (!glfwWindowShouldClose(window)) {
  glClear(GL_COLOR_BUFFER_BIT);
  glBindTexture(GL_TEXTURE_2D, texture);

  try {
    Frame frame = this.grabber.grab();
    if (frame == null) {
      break;
    }
    if (frame.samples != null) {
      final ShortBuffer channelSamplesShortBuffer = (ShortBuffer) frame.samples[0];
      channelSamplesShortBuffer.rewind();

      final ByteBuffer outBuffer =
          ByteBuffer.allocate(channelSamplesShortBuffer.capacity() * 2);

      for (int i = 0; i < channelSamplesShortBuffer.capacity(); i++) {
        short val = channelSamplesShortBuffer.get(i);
        outBuffer.putShort(val);
      }

      soundLine.write(outBuffer.array(), 0, outBuffer.capacity());
      outBuffer.clear();
    }
    if (frame.image != null) {
      glTexSubImage2D(
          GL_TEXTURE_2D,
          0,
          0,
          0,
          this.width,
          this.height,
          GL_BGR,
          GL_UNSIGNED_BYTE,
          (ByteBuffer) frame.image[0]);
    }
  } catch (Exception e1) {
    e1.printStackTrace();
  }
  glBegin(GL_QUADS);
  {
    glTexCoord2f(0.0F, 0.0F);
    glVertex2f(0.0F, 0.0F);
    glTexCoord2f(1.0F, 0.0F);
    glVertex2f(this.width, 0.0F);
    glTexCoord2f(1.0F, 1.0F);
    glVertex2f(this.width, this.height);
    glTexCoord2f(0.0F, 1.0F);
    glVertex2f(0.0F, this.height);
  }
  glEnd();
  glfwSwapBuffers(window);
  glfwPollEvents();
}
glfwFreeCallbacks(window);
glfwDestroyWindow(window);
glfwTerminate();
Objects.requireNonNull(glfwSetErrorCallback(null)).free();

}
}`
Window using Minecraft will have a black screen and video tearing will occur if you create a new window.

[saudet]

It sounds like it doesn't like it when too much processing is done on its thread. Try to use another thread.

[dousha0w0]

But my thread only handles this one thing, could it be a problem with my code?

[dousha0w0]

Actually, I have submitted a task to run my code the audio can be smoothly played, but the screen is black.