Converting classification model from pytorch to onnx to tensorrt , and following tensorrt pynb for inference and using tensorrt 8.6

[Abiganesh]

import numpy as np
import os
import pycuda.driver as cuda
import pycuda.autoinit
import tensorrt as trt

import matplotlib.pyplot as plt
from PIL import Image

TRT_LOGGER = trt.Logger()

# Filenames of TensorRT plan file and input/output images.
engine_file = "efficient64.engine"
# For torchvision models, input images are loaded in to a range of [0, 1] and
# normalized using mean = [0.485, 0.456, 0.406] and stddev = [0.229, 0.224, 0.225].
def preprocess(image):
    # Mean normalization
    mean = np.array([0.485, 0.456, 0.406]).astype('float32')
    stddev = np.array([0.229, 0.224, 0.225]).astype('float32')
    data = (np.asarray(image).astype('float32') / float(255.0) - mean) / stddev
    # Switch from HWC to to CHW order
    return np.moveaxis(data, 2, 0)

def postprocess(data):
    num_classes = 2
    # create a color palette, selecting a color for each class
    palette = np.array([2 ** 25 - 1, 2 ** 15 - 1, 2 ** 21 - 1])
    colors = np.array([palette*i%255 for i in range(num_classes)]).astype("uint8")
    # plot the segmentation predictions for 21 classes in different colors
    img = Image.fromarray(data.astype('uint8'), mode='P')
    img.putpalette(colors)
    return img
def load_engine(engine_file_path):
    assert os.path.exists(engine_file_path)
    print("Reading engine from file {}".format(engine_file_path))
    with open(engine_file_path, "rb") as f, trt.Runtime(TRT_LOGGER) as runtime:
        return runtime.deserialize_cuda_engine(f.read())
def infer(engine, input_file, output_file):
    print("Reading input image from file {}".format(input_file))
    with Image.open(input_file) as img:
        input_image = preprocess(img)
        image_width = img.width
        image_height = img.height

    with engine.create_execution_context() as context:
        # Set input shape based on image dimensions for inference
        print(engine.get_binding_index("input"))
        context.set_binding_shape(engine.get_binding_index("input"), (1, 3, image_height, image_width))
        # Allocate host and device buffers
        bindings = []
        for binding in engine:
            
            binding_idx = engine.get_binding_index(binding)
            print("binding size",context.get_binding_shape(binding_idx))
            size = trt.volume(context.get_binding_shape(binding_idx))
            dtype = trt.nptype(engine.get_binding_dtype(binding))

            if engine.binding_is_input(binding):
                input_buffer = np.ascontiguousarray(input_image)
                input_memory = cuda.mem_alloc(input_image.nbytes)
                print("inpt byte",input_image.nbytes)
                bindings.append(int(input_memory))
            else:
                print("size",size,"dype",dtype)
                output_buffer = cuda.pagelocked_empty(size, dtype)
                print(output_buffer)
                output_memory = cuda.mem_alloc(output_buffer.nbytes)
                print("memory out",output_memory)
                bindings.append(int(output_memory))
        stream = cuda.Stream()
        #host_input = np.array(preprocess_image("2.ppm").numpy(), dtype=np.float32, order='C')
        # Transfer input data to the GPU.
        cuda.memcpy_htod_async(input_memory, input_buffer, stream)
        print("mem ",input_memory,"buff ",input_buffer)
        # Run inference
        a=context.execute_async_v2(bindings=bindings, stream_handle=stream.handle)
        print(a)
        # Transfer prediction output from the GPU.
        cuda.memcpy_dtoh_async(output_buffer, output_memory, stream)
        print(output_buffer)
        print(output_memory)
        # Synchronize the stream
        stream.synchronize()
    

    img =postprocess(np.reshape(output_buffer, (image_height, image_width)))
    
    print("Writing output image to file {}".format(output_file))
    img.convert('RGB').save(output_file, "PPM")
input_file  = "2.ppm"
output_file = "output.ppm"
#plt.imshow(Image.open(input_file))
print("Running TensorRT inference for FCN-ResNet101")
with load_engine(engine_file) as engine:
    infer(engine, input_file, output_file)

I have got below memory error , how to solve that

C:\Users\gj\AppData\Local\Temp\ipykernel_11580\158248581.py:10: DeprecationWarning: Use get_tensor_name instead.
print(engine.get_binding_index("input"))
C:\Users\gj\AppData\Local\Temp\ipykernel_11580\158248581.py:11: DeprecationWarning: Use get_tensor_name instead.
context.set_binding_shape(engine.get_binding_index("input"), (1, 3, image_height, image_width))
C:\Users\gj\AppData\Local\Temp\ipykernel_11580\158248581.py:11: DeprecationWarning: Use set_input_shape instead.
context.set_binding_shape(engine.get_binding_index("input"), (1, 3, image_height, image_width))
C:\Users\gj\AppData\Local\Temp\ipykernel_11580\158248581.py:16: DeprecationWarning: Use get_tensor_name instead.
binding_idx = engine.get_binding_index(binding)
C:\Users\gj\AppData\Local\Temp\ipykernel_11580\158248581.py:17: DeprecationWarning: Use get_tensor_shape instead.
print("binding size",context.get_binding_shape(binding_idx))
C:\Users\gj\AppData\Local\Temp\ipykernel_11580\158248581.py:18: DeprecationWarning: Use get_tensor_shape instead.
size = trt.volume(context.get_binding_shape(binding_idx))
C:\Users\gj\AppData\Local\Temp\ipykernel_11580\158248581.py:19: DeprecationWarning: Use get_tensor_dtype instead.
dtype = trt.nptype(engine.get_binding_dtype(binding))
C:\Users\gj\AppData\Local\Temp\ipykernel_11580\158248581.py:21: DeprecationWarning: Use get_tensor_mode instead.
if engine.binding_is_input(binding):
LogicError Traceback (most recent call last)
Cell In[6], line 3
1 print("Running TensorRT inference for FCN-ResNet101")
2 with load_engine(engine_file) as engine:
----> 3 infer(engine, input_file, output_file)

Cell In[4], line 46, in infer(engine, input_file, output_file)
44 print(output_memory)
45 # Synchronize the stream
---> 46 stream.synchronize()
49 img =postprocess(np.reshape(output_buffer, (image_height, image_width)))
51 print("Writing output image to file {}".format(output_file))

LogicError: cuStreamSynchronize failed: an illegal memory access was encountered
what would bethe problem

[narendasan]

This is not the right repo for this question, we do not use ONNX. Try asking in https://github.com/nvidia/tensorrt