Not able to shift to RAW encoding from jpeg , RGB module

[daksh23182]

1. for the code below :

#include "pmsis.h"

#include "bsp/bsp.h"
#include "bsp/camera/himax.h"
#include "bsp/buffer.h"
#include "gaplib/jpeg_encoder.h"
#include "stdio.h"

#include "cpx.h"
#include "wifi.h"

#define IMG_ORIENTATION 0x0101
#define CAM_WIDTH 320
#define CAM_HEIGHT 224

static pi_task_t task1;
static unsigned char *imgBuff;
static struct pi_device camera;
static pi_buffer_t buffer;

static EventGroupHandle_t evGroup;
#define CAPTURE_DONE_BIT (1 << 0)

// Performance menasuring variables
static uint32_t start = 0;
static uint32_t captureTime = 0;
static uint32_t transferTime = 0;
static uint32_t encodingTime = 0;
// #define OUTPUT_PROFILING_DATA

static int open_pi_camera_himax(struct pi_device *device)
{
    struct pi_himax_conf cam_conf;
    pi_himax_conf_init(&cam_conf);
    cam_conf.format = PI_CAMERA_QVGA;

    pi_open_from_conf(device, &cam_conf);
    if (pi_camera_open(device))
        return -1;

    pi_camera_control(device, PI_CAMERA_CMD_START, 0);
    uint8_t set_value = 0;
    uint8_t reg_value;
    pi_camera_reg_set(device, IMG_ORIENTATION, &set_value);
    pi_time_wait_us(1000000);
    pi_camera_reg_get(device, IMG_ORIENTATION, &reg_value);

    if (set_value != reg_value)
    {
        cpxPrintToConsole(LOG_TO_CRTP, "Failed to rotate camera image\n");
        return -1;
    }

    pi_camera_control(device, PI_CAMERA_CMD_STOP, 0);
    pi_camera_control(device, PI_CAMERA_CMD_AEG_INIT, 0);

    return 0;
}

static int wifiConnected = 0;
static int wifiClientConnected = 0;

static CPXPacket_t rxp;
void rx_task(void *parameters)
{
    while (1)
    {
        cpxReceivePacketBlocking(CPX_F_WIFI_CTRL, &rxp);

        WiFiCTRLPacket_t * wifiCtrl = (WiFiCTRLPacket_t*) rxp.data;

        switch (wifiCtrl->cmd)
        {
            case WIFI_CTRL_STATUS_WIFI_CONNECTED:
                cpxPrintToConsole(LOG_TO_CRTP, "Wifi connected (%u.%u.%u.%u)\n",
                                  wifiCtrl->data[0], wifiCtrl->data[1],
                                  wifiCtrl->data[2], wifiCtrl->data[3]);
                wifiConnected = 1;
                break;
            case WIFI_CTRL_STATUS_CLIENT_CONNECTED:
                cpxPrintToConsole(LOG_TO_CRTP, "Wifi client connection status: %u\n", wifiCtrl->data[0]);
                wifiClientConnected = wifiCtrl->data[0];
                break;
            default:
                break;
        }
    }
}

static void capture_done_cb(void *arg)
{
    xEventGroupSetBits(evGroup, CAPTURE_DONE_BIT);
}

typedef struct
{
    uint8_t magic;
    uint16_t width;
    uint16_t height;
    uint8_t depth;
    uint8_t type;
    uint32_t size;
} __attribute__((packed)) img_header_t;

static jpeg_encoder_t jpeg_encoder;

typedef enum
{
    RAW_ENCODING = 0,
    JPEG_ENCODING = 1
} __attribute__((packed)) StreamerMode_t;

pi_buffer_t header;
uint32_t headerSize;
pi_buffer_t footer;
uint32_t footerSize;
pi_buffer_t jpeg_data;
uint32_t jpegSize;

static StreamerMode_t streamerMode = RAW_ENCODING;

static CPXPacket_t txp;

void createImageHeaderPacket(CPXPacket_t * packet, uint32_t imgSize, StreamerMode_t imgType)
{
    img_header_t *imgHeader = (img_header_t *) packet->data;
    imgHeader->magic = 0xBC;
    imgHeader->width = CAM_WIDTH;
    imgHeader->height = CAM_HEIGHT;
    imgHeader->depth = 1;
    imgHeader->type = imgType;
    imgHeader->size = imgSize;
    packet->dataLength = sizeof(img_header_t);
}

void sendBufferViaCPX(CPXPacket_t * packet, uint8_t * buffer, uint32_t bufferSize)
{
    uint32_t offset = 0;
    uint32_t size = 0;
    do {
        size = sizeof(packet->data);
        if (offset + size > bufferSize)
        {
            size = bufferSize - offset;
        }
        memcpy(packet->data, &buffer[offset], sizeof(packet->data));
        packet->dataLength = size;
        cpxSendPacketBlocking(packet);
        offset += size;
    } while (size == sizeof(packet->data));
}

#ifdef SETUP_WIFI_AP
void setupWiFi(void)
{
    static char ssid[] = "WiFi streaming example";
    cpxPrintToConsole(LOG_TO_CRTP, "Setting up WiFi AP\n");
    txp.route.destination = CPX_T_ESP32;
    rxp.route.source = CPX_T_GAP8;
    txp.route.function = CPX_F_WIFI_CTRL;
    txp.route.version = CPX_VERSION;
    WiFiCTRLPacket_t * wifiCtrl = (WiFiCTRLPacket_t*) txp.data;

    wifiCtrl->cmd = WIFI_CTRL_SET_SSID;
    memcpy(wifiCtrl->data, ssid, sizeof(ssid));
    txp.dataLength = sizeof(ssid);
    cpxSendPacketBlocking(&txp);

    wifiCtrl->cmd = WIFI_CTRL_WIFI_CONNECT;
    wifiCtrl->data[0] = 0x01;
    txp.dataLength = 2;
    cpxSendPacketBlocking(&txp);
}
#endif

void camera_task(void *parameters)
{
    vTaskDelay(2000);

#ifdef SETUP_WIFI_AP
    setupWiFi();
#endif

    cpxPrintToConsole(LOG_TO_CRTP, "Starting camera task...\n");

    uint32_t resolution = CAM_WIDTH * CAM_HEIGHT;
    uint32_t captureSize = resolution * sizeof(unsigned char);
    imgBuff = (unsigned char *)pmsis_l2_malloc(captureSize);
    if (imgBuff == NULL)
    {
        cpxPrintToConsole(LOG_TO_CRTP, "Failed to allocate Memory for Image \n");
        return;
    }

    if (open_pi_camera_himax(&camera))
    {
        cpxPrintToConsole(LOG_TO_CRTP, "Failed to open camera\n");
        return;
    }

    pi_buffer_init(&buffer, PI_BUFFER_TYPE_L2, imgBuff);
    pi_buffer_set_format(&buffer, CAM_WIDTH, CAM_HEIGHT, 1, PI_BUFFER_FORMAT_GRAY);

    cpxInitRoute(CPX_T_GAP8, CPX_T_WIFI_HOST, CPX_F_APP, &txp.route);

    uint32_t imgSize = 0;

    while (1)
    {
        if (wifiClientConnected == 1)
        {
            start = xTaskGetTickCount();
            pi_camera_capture_async(&camera, imgBuff, resolution, pi_task_callback(&task1, capture_done_cb, NULL));
            pi_camera_control(&camera, PI_CAMERA_CMD_START, 0);
            xEventGroupWaitBits(evGroup, CAPTURE_DONE_BIT, pdTRUE, pdFALSE, (TickType_t)portMAX_DELAY);
            pi_camera_control(&camera, PI_CAMERA_CMD_STOP, 0);
            captureTime = xTaskGetTickCount() - start;

            imgSize = captureSize;

            createImageHeaderPacket(&txp, imgSize, RAW_ENCODING);
            cpxSendPacketBlocking(&txp);

            sendBufferViaCPX(&txp, imgBuff, imgSize);
        }
        else
        {
            vTaskDelay(10);
        }
    }
}

#define LED_PIN 2
static pi_device_t led_gpio_dev;

void hb_task(void *parameters)
{
    (void)parameters;
    const TickType_t xDelay = 500 / portTICK_PERIOD_MS;

    while (1)
    {
        pi_gpio_pin_write(&led_gpio_dev, LED_PIN, 1);
        vTaskDelay(xDelay);
        pi_gpio_pin_write(&led_gpio_dev, LED_PIN, 0);
        vTaskDelay(xDelay);
    }
}

void start_example(void)
{
    struct pi_uart_conf conf;
    struct pi_device device;
    pi_uart_conf_init(&conf);
    conf.baudrate_bps = 115200;

    pi_open_from_conf(&device, &conf);
    if (pi_uart_open(&device))
    {
        printf("[UART] open failed !\n");
        pmsis_exit(-1);
    }

    cpxInit();
    cpxEnableFunction(CPX_F_WIFI_CTRL);

    cpxPrintToConsole(LOG_TO_CRTP, "-- WiFi image streamer example --\n");

    evGroup = xEventGroupCreate();

    xTaskCreate(hb_task, "hb_task", configMINIMAL_STACK_SIZE * 2, NULL, tskIDLE_PRIORITY + 1, NULL);
    xTaskCreate(camera_task, "camera_task", configMINIMAL_STACK_SIZE * 4, NULL, tskIDLE_PRIORITY + 1, NULL);
    xTaskCreate(rx_task, "rx_task", configMINIMAL_STACK_SIZE * 2, NULL, tskIDLE_PRIORITY + 1, NULL);

    while (1)
    {
        pi_yield();
    }
}

int main(void)
{
    pi_bsp_init();
    pi_freq_set(PI_FREQ_DOMAIN_FC, 250000000);
    __pi_pmu_voltage_set(PI_PMU_DOMAIN_FC, 1200);
    return pmsis_kickoff((void *)start_example);
}

this is the file from the repo, i changed the line static StreamerMode_t streamerMode = RAW_ENCODING; (JPEG_ENCODING to RAW_ENCODING) and still on running : python opencv-viewer.py , the pop up window shows JPEG on top, how to shift?

2. also the output images in the window seems to be rotated along with trying to show 3 frames together - how to fix this as well?
3. after switching to the rgb camera module , i added these in the makefile :
   APP_CFLAGS += -DCOLOR_IMAGE
   PP_CFLAGS += -DQVGA_MODE, but then on running the python command the output is plain black.

[gemenerik]

I still see the line createImageHeaderPacket(&txp, imgSize, RAW_ENCODING);, have you modified that as well?

[daksh23182]

I still see the line createImageHeaderPacket(&txp, imgSize, RAW_ENCODING);, have you modified that as well?

what do i need to modify in this line? i need the raw encoding

[gemenerik]

Sorry, I misunderstood what you are trying to do. The default is RAW encoding. Revert to the original file and also remove the compile flags you added.

Looking at the diff between your file and the source, you have actually made some additional changes including changing the resolution and rotation of the image. That very likely explains the rotation and distortion.

The viewer should automatically handle JPEG vs RAW encoding.

Let me know if it works.