USB UVC get still images larger than 640x480px (IEC-5)

[dmitrij9992905]

Hello everyone!

I see that one of limitations of USB UVC lib is maximum resolution of 640x480px 15 FPS.
But I need the still image larger than 640x480 but supported by the camera and fits into ESP32-PSRAM.

Is it possible to implement on ESP32-S2(S3)?

[dmitrij9992905]

Hello everyone!
I managed to get JPEG frames with converting with "conversions" component from ESP32-Camera, but I managed to get solid images in resolution 160120 px.
I'd like to get still images with resolution more than 160120 (ideally - HD-images).
I'm trying to call uvc_trigger_still, but camera suspends on some time and there are images as usual (no frame with frame resolution higher).
Could you please help me to get still images?
Here is the code:

#include "esp_err.h"
#include "esp_log.h"
#include "esp_timer.h"
#include "http_server.hpp"
#include <string.h>
#include <sys/unistd.h>
#include <sys/stat.h>
#include "esp_vfs_fat.h"
#include "sdmmc_cmd.h"
#include "img_converters.h"

#ifdef CONFIG_USE_PSRAM
#if CONFIG_IDF_TARGET_ESP32 // ESP32/PICO-D4
#include "esp32/spiram.h"
#elif CONFIG_IDF_TARGET_ESP32S2
#include "esp32s2/spiram.h"
#elif CONFIG_IDF_TARGET_ESP32S3
#include "esp32s3/spiram.h"
#else
#error Target CONFIG_IDF_TARGET is not supported
#endif
#endif

#define MOUNT_POINT "/sdcard"

#define PIN_NUM_MISO 10
#define PIN_NUM_MOSI 11
#define PIN_NUM_CLK  12
#define PIN_NUM_CS   13

// #if CONFIG_IDF_TARGET_ESP32S2 | CONFIG_IDF_TARGET_ESP32S3
// #define SPI_DMA_CHAN    host.slot
// #elif CONFIG_IDF_TARGET_ESP32C3
#define SPI_DMA_CHAN    SPI_DMA_CH_AUTO
// #else
// #define SPI_DMA_CHAN    1
// #endif

#define STILL_TRIGGER_PIN   0

#define FPS 5
#define PREVIEW_WIDTH 160 
#define PREVIEW_HEIGHT 120
#define CAPTURE_WIDTH 320
#define CAPTURE_HEIGHT 240
#define FORMAT UVC_COLOR_FORMAT_YUYV // UVC_COLOR_FORMAT_MJPEG

// Attached camera can be filtered out based on (non-zero value of) PID, VID, SERIAL_NUMBER
#define PID 0
#define VID 0
#define SERIAL_NUMBER NULL

#define UVC_CHECK(exp) do {                 \
    uvc_error_t _err_ = (exp);              \
    if(_err_ < 0) {                         \
        ESP_LOGE(TAG, "UVC error: %s",      \
                 uvc_error_string(_err_));  \
        assert(0);                          \
    }                                       \
} while(0)

static const char *TAG = "MAIN";

static SemaphoreHandle_t ready_to_uninstall_usb;
static EventGroupHandle_t app_flags;
bool frame_got = false;

uvc_frame_t *frame_to_copy;

// Handles common USB host library events
static void usb_lib_handler_task(void *args)
{
    while (1) {
        uint32_t event_flags;
        usb_host_lib_handle_events(portMAX_DELAY, &event_flags);
        // Release devices once all clients has deregistered
        if (event_flags & USB_HOST_LIB_EVENT_FLAGS_NO_CLIENTS) {
            usb_host_device_free_all();
        }
        // Give ready_to_uninstall_usb semaphore to indicate that USB Host library
        // can be deinitialized, and terminate this task.
        if (event_flags & USB_HOST_LIB_EVENT_FLAGS_ALL_FREE) {
            xSemaphoreGive(ready_to_uninstall_usb);
        }
    }

    vTaskDelete(NULL);
}

static esp_err_t initialize_usb_host_lib(void)
{
    TaskHandle_t task_handle = NULL;

    const usb_host_config_t host_config = {
        .intr_flags = ESP_INTR_FLAG_LEVEL1
    };

    esp_err_t err = usb_host_install(&host_config);
    if (err != ESP_OK) {
        return err;
    }

    ready_to_uninstall_usb = xSemaphoreCreateBinary();
    if (ready_to_uninstall_usb == NULL) {
        usb_host_uninstall();
        return ESP_ERR_NO_MEM;
    }

    if (xTaskCreate(usb_lib_handler_task, "usb_events", 8192, NULL, 2, &task_handle) != pdPASS) {
        vSemaphoreDelete(ready_to_uninstall_usb);
        usb_host_uninstall();
        return ESP_ERR_NO_MEM;
    }

    return ESP_OK;
}

static void uninitialize_usb_host_lib(void)
{
    xSemaphoreTake(ready_to_uninstall_usb, portMAX_DELAY);
    vSemaphoreDelete(ready_to_uninstall_usb);

    if ( usb_host_uninstall() != ESP_OK) {
        ESP_LOGE(TAG, "Failed to uninstall usb_host");
    }
}

static void libuvc_adapter_cb(libuvc_adapter_event_t event)
{
    xEventGroupSetBits(app_flags, event);
}

/* This callback function runs once per frame. Use it to perform any
 * quick processing you need, or have it put the frame into your application's
 * input queue. If this function takes too long, you'll start losing frames. */
void cb(uvc_frame_t *frame, void *ptr) {
    uvc_frame_t *bgr;
    uvc_frame_t *convertable;
    uvc_error_t ret;
    enum uvc_frame_format *frame_format = (enum uvc_frame_format *)ptr;
    FILE *fp;
    static int jpeg_count = 0;
    static int yuyv_count = 0;
    static const char *H264_FILE = "iOSDevLog.h264";
    static const char *MJPEG_FILE = ".jpeg";
    bool conversion_to_jpeg_success = false;
    uint8_t * jpg_out = nullptr;
    size_t jpg_out_size = 0;
    char filename[48];

    /* We'll convert the image from YUV/JPEG to BGR, so allocate space */
    bgr = uvc_allocate_frame(frame->width * frame->height * 3);
    if (!bgr) {
        printf("unable to allocate bgr frame!\n");
        return;
    }

    printf("callback! frame_format = %d, width = %d, height = %d, length = %d, ptr = %d\n",
        frame->frame_format, frame->width, frame->height, frame->data_bytes, (int) ptr);

    switch (frame->frame_format) {
    case UVC_FRAME_FORMAT_H264:
        /* use `ffplay H264_FILE` to play */
        /* fp = fopen(H264_FILE, "a");
        * fwrite(frame->data, 1, frame->data_bytes, fp);
        * fclose(fp); */
        break;
    case UVC_COLOR_FORMAT_MJPEG:
        // ret = uvc_mjpeg_convert(frame, bgr);
        // if (ret) {
        //     uvc_perror(ret, "uvc_mjpeg_convert");
        // }
        // else {
        //     sprintf(filename, "%s/%d%s", MOUNT_POINT, jpeg_count++, MJPEG_FILE);
        //     fp = fopen(filename, "w");
        //     if (fp == NULL) {
        //         ESP_LOGE(TAG, "Failed to open file for writing: %s", filename);
        //         // fclose(fp);
        //         // return;
        //     }
        //     else {
        //         fwrite(frame->data, 1, frame->data_bytes, fp);
        //         fclose(fp);
        //     }
        // // }
        // //uvc_free_frame(bgr);
        // return;
        // frame_to_copy = uvc_allocate_frame(frame->width * frame->height * 3);
        // memcpy(frame_to_copy, frame, (frame->width * frame->height * 3));
        // frame_got = true;

        conversion_to_jpeg_success = fmt2jpg((uint8_t*)frame->data, frame->data_bytes, PREVIEW_WIDTH, PREVIEW_HEIGHT, pixformat_t::PIXFORMAT_YUV422, 75, &jpg_out, &jpg_out_size);
        if (conversion_to_jpeg_success) {
            sprintf(filename, "%s/%d%s", MOUNT_POINT, jpeg_count++, ".jpg");
            fp = fopen(filename, "w");
            if (fp == NULL) {
                ESP_LOGE(TAG, "Failed to open file for writing: %s", filename);
                // fclose(fp);
                // return;
            }
            else {
                ESP_LOGI(TAG, "Writing to %s...", filename);
                fwrite(jpg_out, 1, jpg_out_size, fp);
                fclose(fp);
            }
            free(jpg_out);
        }
        else {
            ESP_LOGE(TAG, "Failed to convert to JPEG");
        }

        break;
    case UVC_COLOR_FORMAT_YUYV:
        ESP_LOGI(TAG, "Got YUYV frame");
        /* Do the RGB conversion */
        // ret = uvc_any2rgb(frame, bgr);
        // if (ret) {
        //     uvc_perror(ret, "uvc_any2rgb");
        //     uvc_free_frame(bgr);
        //     return;
        // }
        // else {
        
        // Do the conversion to JPEG
        conversion_to_jpeg_success = fmt2jpg((uint8_t*)frame->data, frame->data_bytes, PREVIEW_WIDTH, PREVIEW_HEIGHT, pixformat_t::PIXFORMAT_YUV422, 75, &jpg_out, &jpg_out_size);
        if (conversion_to_jpeg_success) {
            sprintf(filename, "%s/%d%s", MOUNT_POINT, yuyv_count++, ".jpg");
            fp = fopen(filename, "w");
            if (fp == NULL) {
                ESP_LOGE(TAG, "Failed to open file for writing: %s", filename);
                // fclose(fp);
                // return;
            }
            else {
                ESP_LOGI(TAG, "Writing to %s...", filename);
                fwrite(jpg_out, 1, jpg_out_size, fp);
                fclose(fp);
            }
            free(jpg_out);
        }
        else {
            ESP_LOGE(TAG, "Failed to convert to JPEG");
        }
        // }
        break;
    default:
        break;
    }

    if (frame->sequence % 30 == 0) {
        printf(" * got image %u\n",  frame->sequence);
    }

  /* Call a user function:
   *
   * my_type *my_obj = (*my_type) ptr;
   * my_user_function(ptr, bgr);
   * my_other_function(ptr, bgr->data, bgr->width, bgr->height);
   */

    uvc_free_frame(bgr);
}

static EventBits_t wait_for_event(EventBits_t event)
{
    return xEventGroupWaitBits(app_flags, event, pdTRUE, pdFALSE, portMAX_DELAY) & event;
}

void show_heap(void * params) {
    while(1) {
        ESP_LOGI(TAG, "Heap size: %d", esp_get_free_heap_size());
        vTaskDelay(1000 / portTICK_PERIOD_MS);
    }
    vTaskDelete(NULL);
}

extern "C" void app_main(void)
{
    uvc_context_t *ctx;
    uvc_device_t *dev;
    uvc_device_handle_t *devh;
    uvc_stream_ctrl_t stream_ctrl;
    uvc_still_ctrl_t still_ctrl;
    uvc_error_t res;
    uint8_t still_button_state = 0;
    uint8_t still_button_state_old = 0;
    bool supports_still = true;
    int still_negotiations = 0;

    app_flags = xEventGroupCreate();
    assert(app_flags);
    
    xTaskCreate(
        show_heap,
        "show_heap",
        4096,
        NULL,
        2,
        NULL);

    // xTaskCreate(
    //     write_to_sdcard,
    //     "write_to_sd",
    //     4096,
    //     NULL,
    //     2,
    //     NULL);

    gpio_reset_pin((gpio_num_t)STILL_TRIGGER_PIN);
	gpio_set_direction((gpio_num_t)STILL_TRIGGER_PIN, GPIO_MODE_INPUT);
	gpio_set_pull_mode((gpio_num_t)STILL_TRIGGER_PIN, GPIO_PULLUP_ONLY);

    libuvc_adapter_config_t config = {
        .create_background_task = true,
        .task_priority = 5,
        .stack_size = 180000,
        .callback = libuvc_adapter_cb
    };

    ESP_ERROR_CHECK( initialize_usb_host_lib() );

    libuvc_adapter_set_config(&config);

    UVC_CHECK( uvc_init(&ctx, NULL) );

    esp_err_t ret;

    // Options for mounting the filesystem.
    // If format_if_mount_failed is set to true, SD card will be partitioned and
    // formatted in case when mounting fails.
    esp_vfs_fat_sdmmc_mount_config_t mount_config = {
#ifdef CONFIG_EXAMPLE_FORMAT_IF_MOUNT_FAILED
        .format_if_mount_failed = true,
#else
        .format_if_mount_failed = false,
#endif // EXAMPLE_FORMAT_IF_MOUNT_FAILED
        .max_files = 5,
        .allocation_unit_size = 4 * 1024
    };
    sdmmc_card_t *card;
    const char mount_point[] = MOUNT_POINT;
    ESP_LOGI(TAG, "Initializing SD card");

    // Use settings defined above to initialize SD card and mount FAT filesystem.
    // Note: esp_vfs_fat_sdmmc/sdspi_mount is all-in-one convenience functions.
    // Please check its source code and implement error recovery when developing
    // production applications.
    ESP_LOGI(TAG, "Using SPI peripheral");

    sdmmc_host_t host = SDSPI_HOST_DEFAULT();
    spi_bus_config_t bus_cfg = {
        .mosi_io_num = (gpio_num_t)PIN_NUM_MOSI,
        .miso_io_num = (gpio_num_t)PIN_NUM_MISO,
        .sclk_io_num = (gpio_num_t)PIN_NUM_CLK,
        .quadwp_io_num = -1,
        .quadhd_io_num = -1,
        .max_transfer_sz = 4000,
    };
    ret = spi_bus_initialize((spi_host_device_t)host.slot, &bus_cfg, SPI_DMA_CHAN);
    if (ret != ESP_OK) {
        ESP_LOGE(TAG, "Failed to initialize bus.");
        return;
    }

    // This initializes the slot without card detect (CD) and write protect (WP) signals.
    // Modify slot_config.gpio_cd and slot_config.gpio_wp if your board has these signals.
    sdspi_device_config_t slot_config = SDSPI_DEVICE_CONFIG_DEFAULT();
    slot_config.gpio_cs = (gpio_num_t)PIN_NUM_CS;
    slot_config.host_id = (spi_host_device_t)host.slot;

    ESP_LOGI(TAG, "Mounting filesystem");
    ret = esp_vfs_fat_sdspi_mount(mount_point, &host, &slot_config, &mount_config, &card);

    if (ret != ESP_OK) {
        if (ret == ESP_FAIL) {
            ESP_LOGE(TAG, "Failed to mount filesystem. "
                     "If you want the card to be formatted, set the EXAMPLE_FORMAT_IF_MOUNT_FAILED menuconfig option.");
        } else {
            ESP_LOGE(TAG, "Failed to initialize the card (%s). "
                     "Make sure SD card lines have pull-up resistors in place.", esp_err_to_name(ret));
        }
        return;
    }
    ESP_LOGI(TAG, "Filesystem mounted");

    // Card has been initialized, print its properties
    sdmmc_card_print_info(stdout, card);

    while(1) {
        // First create a file.
        const char *file_hello = MOUNT_POINT"/hello.txt";

        ESP_LOGI(TAG, "Opening file %s", file_hello);
        FILE *f = fopen(file_hello, "w");
        if (f == NULL) {
            ESP_LOGE(TAG, "Failed to open file for writing");
            return;
        }
        fprintf(f, "Hello %s!\n", card->cid.name);
        fclose(f);
        ESP_LOGI(TAG, "File written");
        printf("Waiting for device\n");
        wait_for_event(UVC_DEVICE_CONNECTED);

        UVC_CHECK( uvc_find_device(ctx, &dev, PID, VID, SERIAL_NUMBER) );
        puts("Device found");

        UVC_CHECK( uvc_open(dev, &devh) );

        // Uncomment to print configuration descriptor
        // libuvc_adapter_print_descriptors(devh);

        // uvc_set_button_callback(devh, button_callback, NULL);
        
        // Print known device information
        // uvc_print_diag(devh, stderr);

        const uvc_format_desc_t *format_desc = uvc_get_format_descs(devh);
        const uvc_frame_desc_t *frame_desc = format_desc->frame_descs;
        enum uvc_frame_format frame_format;
        // int width = 640;
        // int height = 480;
        // int fps = 30;

        // switch (format_desc->bDescriptorSubtype) {
        //     case UVC_VS_FORMAT_MJPEG:
        //         frame_format = UVC_COLOR_FORMAT_MJPEG;
        //         break;
        //     case UVC_VS_FORMAT_FRAME_BASED:
        //         frame_format = UVC_FRAME_FORMAT_H264;
        //         break;
        //     default:
        //         frame_format = UVC_FRAME_FORMAT_YUYV;
        //         break;
        // }

        // if (frame_desc) {
        //     width = frame_desc->wWidth;
        //     height = frame_desc->wHeight;
        //     fps = 10000000 / frame_desc->dwDefaultFrameInterval;
        // }

        // printf("\nFirst format: (%4s) %dx%d %dfps\n", format_desc->fourccFormat, width, height, fps);

        // Negotiate stream profile
        res = uvc_get_stream_ctrl_format_size(devh, &stream_ctrl, FORMAT, PREVIEW_WIDTH, PREVIEW_HEIGHT, FPS );
        //res = uvc_get_stream_ctrl_format_size(devh, &stream_ctrl, frame_format, width, height, fps );
        while (res != UVC_SUCCESS) {
            printf("Negotiating streaming format failed, trying again...\n");
            res = uvc_get_stream_ctrl_format_size(devh, &stream_ctrl, FORMAT, PREVIEW_WIDTH, PREVIEW_HEIGHT, FPS );
            //res = uvc_get_stream_ctrl_format_size(devh, &stream_ctrl, frame_format, width, height, fps );
            sleep(1);
        }
        
        // Negotiate still profile
        // res = uvc_probe_still_ctrl(devh, &still_ctrl);
        // if (res != UVC_SUCCESS) {
        //     ESP_LOGW(TAG, "This camera doesn't support still mode");
        //     supports_still = false;
        // }
        // else {
        still_negotiations = 0;
        supports_still = true;
        res = uvc_get_still_ctrl_format_size(devh, &stream_ctrl, &still_ctrl, CAPTURE_WIDTH, CAPTURE_HEIGHT);
        // res = uvc_get_still_ctrl_format_size(devh, nullptr, &still_ctrl, CAPTURE_WIDTH, CAPTURE_HEIGHT);
        while (res != UVC_SUCCESS) {
            printf("Negotiating still format failed, trying again...\n");
            res = uvc_get_still_ctrl_format_size(devh, &stream_ctrl, &still_ctrl, CAPTURE_WIDTH, CAPTURE_HEIGHT);
            sleep(1);
            still_negotiations++;
            if (still_negotiations > 2) {
                supports_still = false;
                ESP_LOGW(TAG, "This camera doesn't support still mode: %dx%d", CAPTURE_WIDTH, CAPTURE_HEIGHT);
                break;
            }
        }
            // supports_still = true;
        // }
        

        // dwMaxPayloadTransferSize has to be overwritten to MPS (maximum packet size)
        // supported by ESP32-S2(S3), as libuvc selects the highest possible MPS by default.
        stream_ctrl.dwMaxPayloadTransferSize = 512;
        still_ctrl.dwMaxPayloadTransferSize = 512;

        uvc_print_stream_ctrl(&stream_ctrl, stderr);

        // vTaskDelay(5000 / portTICK_PERIOD_MS);

        // if (dev == nullptr) {
        //     ESP_LOGE(TAG, "Device handler was NULL");
        // }
        // if (devh == nullptr) {
        //     ESP_LOGE(TAG, "Device UVC handler was NULL");
        // }
        if (res < 0) {
            uvc_perror(res, "get_mode"); /* device doesn't provide a matching stream */
        } else {
            /* Start the video stream. The library will call user function cb:
            *   cb(frame, (void *) 12345)
            */
            res = uvc_start_streaming(devh, &stream_ctrl, cb, (void *) 1234, 0);

            if (res < 0) {
            uvc_perror(res, "start_streaming"); /* unable to start stream */
            } else {
            puts("Streaming...");

            /* enable auto exposure - see uvc_set_ae_mode documentation */
            puts("Enabling auto exposure ...");
            const uint8_t UVC_AUTO_EXPOSURE_MODE_AUTO = 2;
            res = uvc_set_ae_mode(devh, UVC_AUTO_EXPOSURE_MODE_AUTO);
            if (res == UVC_SUCCESS) {
                puts(" ... enabled auto exposure");
            } else if (res == UVC_ERROR_PIPE) {
                /* this error indicates that the camera does not support the full AE mode;
                * try again, using aperture priority mode (fixed aperture, variable exposure time) */
                puts(" ... full AE not supported, trying aperture priority mode");
                const uint8_t UVC_AUTO_EXPOSURE_MODE_APERTURE_PRIORITY = 8;
                res = uvc_set_ae_mode(devh, UVC_AUTO_EXPOSURE_MODE_APERTURE_PRIORITY);
                if (res < 0) {
                uvc_perror(res, " ... uvc_set_ae_mode failed to enable aperture priority mode");
                } else {
                puts(" ... enabled aperture priority auto exposure mode");
                }
            } else {
                uvc_perror(res, " ... uvc_set_ae_mode failed to enable auto exposure mode");
            }

            while(1) {
                still_button_state = gpio_get_level((gpio_num_t)STILL_TRIGGER_PIN);
                if (still_button_state != still_button_state_old) {
                    still_button_state_old = still_button_state;
                    if (still_button_state == 0 && supports_still) {
                        ESP_LOGI(TAG, "Triggering still");
                        // uvc_stop_streaming(devh);
                        uvc_trigger_still(devh, &still_ctrl);
                        // uvc_start_streaming(devh, &stream_ctrl, cb, (void *) 1234, 0);
                    }
                }
                vTaskDelay(100 / portTICK_PERIOD_MS); /* stream for 10 seconds */
            }
                

            // uvc_stop_streaming(devh);

            // res = uvc_get_still_ctrl_format_size(devh, &stream_ctrl, &still_ctrl, CAPTURE_WIDTH, CAPTURE_HEIGHT);
            // // res = uvc_get_still_ctrl_format_size(devh, nullptr, &still_ctrl, CAPTURE_WIDTH, CAPTURE_HEIGHT);
            // while (res != UVC_SUCCESS) {
            //     printf("Negotiating still format failed, trying again...\n");
            //     res = uvc_get_still_ctrl_format_size(devh, &stream_ctrl, &still_ctrl, CAPTURE_WIDTH, CAPTURE_HEIGHT);
            //     sleep(1);
            // }

            // while(1) {
            //     uvc_trigger_still(devh, &still_ctrl);
            //     vTaskDelay(10000 / portTICK_PERIOD_MS);
            // }

            /* End the stream. Blocks until last callback is serviced */
            uvc_stop_streaming(devh);
            puts("Done streaming.");
            }
        }

        uvc_close(devh);

        vTaskDelay(1000 / portTICK_PERIOD_MS);
    }

    uvc_exit(ctx);
    puts("UVC exited");

    uninitialize_usb_host_lib();
    // All done, unmount partition and disable SPI peripheral
    esp_vfs_fat_sdcard_unmount(mount_point, card);
    ESP_LOGI(TAG, "Card unmounted");

    //deinitialize the bus after all devices are removed
    spi_bus_free((spi_host_device_t)host.slot);

    return;
}

The camera supports:

DEVICE CONFIGURATION (0c45:6340/        S) ---
Status: idle
VideoControl:
        bcdUVC: 0x0100
VideoStreaming(1):
        bEndpointAddress: 129
        Formats:
        UncompressedFormat(1)
                  bits per pixel: 16
                  GUID: 5955593200001000800000aa00389b71 (YUY2)
                  default frame: 1
                  aspect ratio: 0x0
                  interlace flags: 00
                  copy protect: 00
                        FrameDescriptor(1)
                          capabilities: 00
                          size: 320x240
                          bit rate: 6144000-6144000
                          max frame size: 153600
                          default interval: 1/5
                          interval[0]: 1/5
                        FrameDescriptor(2)
                          capabilities: 00
                          size: 160x120
                          bit rate: 1536000-6144000
                          max frame size: 38400
                          default interval: 1/20
                          interval[0]: 1/20
                          interval[1]: 1/15
                          interval[2]: 1/10
                          interval[3]: 1/5
                        StillFrameDescriptor
                          bEndPointAddress: 00
                          wWidth(1) = 320
                          wHeight(1) = 240
                          wWidth(2) = 160
                          wHeight(2) = 120
END DEVICE CONFIGURATION