YUYV格式推流

.vscode/settings.json

@@ -60,6 +60,7 @@
         "stop_token": "cpp",
         "streambuf": "cpp",
         "thread": "cpp",
-        "typeinfo": "cpp"
+        "typeinfo": "cpp",
+        "chrono": "cpp"
     }
 }

Vimba_6_0/VimbaCPP/Examples/AsynchronousGrab/VirtualCameraColor/Build/Make/Makefile

@@ -34,7 +34,8 @@ DEFINES             =
 
 CFLAGS              = $(COMMON_CFLAGS) \
                       $(VIMBACPP_CFLAGS) \
-                      $(VIMBAIMAGETRANSFORM_CFLAGS)
+                      $(VIMBAIMAGETRANSFORM_CFLAGS) \
+                      
 
 OBJ_FILES           = $(OBJ_DIR)/ApiController.o \
                       $(OBJ_DIR)/FrameObserver.o \

Vimba_6_0/VimbaCPP/Examples/AsynchronousGrab/VirtualCameraColor/Source/FrameObserver.cpp

@@ -4,6 +4,8 @@
 #include <sys/ioctl.h>
 #include <unistd.h>
 #include <chrono>
+#include <vector>
+#include <iostream> 
 
 namespace AVT {
 namespace VmbAPI {
@@ -36,8 +38,9 @@ bool FrameObserver::SetupVideoDevice()
     
     m_vfmt.fmt.pix.width = 640;
     m_vfmt.fmt.pix.height = 480;
-    m_vfmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV;
+    m_vfmt.fmt.pix.pixelformat = V4L2_PIX_FMT_RGB24;  // 改为RGB24格式
     m_vfmt.fmt.pix.field = V4L2_FIELD_NONE;
+    m_vfmt.fmt.pix.sizeimage = 640 * 480 * 3;  // RGB24是3字节/像素
     
     if (ioctl(m_videoFd, VIDIOC_S_FMT, &m_vfmt) < 0) {
         perror("Set video format");
@@ -76,8 +79,8 @@ bool FrameObserver::UpdateVideoFormat(VmbUint32_t width, VmbUint32_t height, Vmb
             m_vfmt.fmt.pix.sizeimage = width * height;
             break;
         case VmbPixelFormatBayerRG8:
-            m_vfmt.fmt.pix.pixelformat = V4L2_PIX_FMT_SRGGB8;
-            m_vfmt.fmt.pix.sizeimage = width * height;
+            m_vfmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV;  // We convert Bayer to YUYV
+            m_vfmt.fmt.pix.sizeimage = width * height * 2;
             break;
         default:
             m_vfmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV;
@@ -92,10 +95,64 @@ bool FrameObserver::UpdateVideoFormat(VmbUint32_t width, VmbUint32_t height, Vmb
     return true;
 }
 
-void FrameObserver::FrameReceived(const FramePtr pFrame) 
-{
+// Convert Bayer RGGB to RGB (simplified demosaicing)
+void FrameObserver::ConvertBayerRGGBToRGB(const VmbUchar_t* bayerData, VmbUchar_t* rgbData, VmbUint32_t width, VmbUint32_t height) {
+    // 只处理中心像素（避免边界越界）
+    for (VmbUint32_t y = 1; y < height - 1; y++) {
+        for (VmbUint32_t x = 1; x < width - 1; x++) {
+            VmbUint32_t idx = y * width + x;
+            VmbUchar_t r, g, b;
+
+            if (y % 2 == 0) {  // Even row (RGRG...)
+                if (x % 2 == 0) {  // R
+                    r = bayerData[idx];
+                    g = (bayerData[idx - 1] + bayerData[idx + 1] + bayerData[idx - width] + bayerData[idx + width]) / 4;
+                    b = (bayerData[idx - width - 1] + bayerData[idx - width + 1] + bayerData[idx + width - 1] + bayerData[idx + width + 1]) / 4;
+                } else {  // G
+                    r = (bayerData[idx - 1] + bayerData[idx + 1]) / 2;
+                    g = bayerData[idx];
+                    b = (bayerData[idx - width] + bayerData[idx + width]) / 2;
+                }
+            } else {  // Odd row (GBGB...)
+                if (x % 2 == 0) {  // G
+                    r = (bayerData[idx - width] + bayerData[idx + width]) / 2;
+                    g = bayerData[idx];
+                    b = (bayerData[idx - 1] + bayerData[idx + 1]) / 2;
+                } else {  // B
+                    r = (bayerData[idx - width - 1] + bayerData[idx - width + 1] + bayerData[idx + width - 1] + bayerData[idx + width + 1]) / 4;
+                    g = (bayerData[idx - 1] + bayerData[idx + 1] + bayerData[idx - width] + bayerData[idx + width]) / 4;
+                    b = bayerData[idx];
+                }
+            }
+
+            rgbData[idx * 3] = r;
+            rgbData[idx * 3 + 1] = g;
+            rgbData[idx * 3 + 2] = b;
+        }
+    }
+}
+// Convert RGB to YUYV (4:2:2)
+void FrameObserver::ConvertRGBToYUYV(const VmbUchar_t* rgbData, VmbUchar_t* yuyvData, VmbUint32_t width, VmbUint32_t height) {
+    for (VmbUint32_t i = 0; i < width * height; i += 2) {
+        VmbUchar_t r1 = rgbData[i * 3], g1 = rgbData[i * 3 + 1], b1 = rgbData[i * 3 + 2];
+        VmbUchar_t r2 = rgbData[(i + 1) * 3], g2 = rgbData[(i + 1) * 3 + 1], b2 = rgbData[(i + 1) * 3 + 2];
+
+        // YUV转换（简化版BT.601）
+        VmbUchar_t y1 = 0.299f * r1 + 0.587f * g1 + 0.114f * b1;
+        VmbUchar_t u1 = -0.147f * r1 - 0.289f * g1 + 0.436f * b1 + 128;
+        VmbUchar_t v1 = 0.615f * r1 - 0.515f * g1 - 0.100f * b1 + 128;
+        VmbUchar_t y2 = 0.299f * r2 + 0.587f * g2 + 0.114f * b2;
+
+        yuyvData[i * 2] = y1;
+        yuyvData[i * 2 + 1] = u1;
+        yuyvData[i * 2 + 2] = y2;
+        yuyvData[i * 2 + 3] = v1;
+    }
+}
+void FrameObserver::FrameReceived(const FramePtr pFrame) {
     auto startTime = std::chrono::high_resolution_clock::now();
     
+    // 1. 获取帧数据
     VmbUchar_t* pBuffer = nullptr;
     VmbUint32_t nSize = 0;
     VmbUint32_t width = 0, height = 0;
@@ -106,26 +163,57 @@ void FrameObserver::FrameReceived(const FramePtr pFrame)
         pFrame->GetWidth(width) != VmbErrorSuccess ||
         pFrame->GetHeight(height) != VmbErrorSuccess ||
         pFrame->GetPixelFormat(pixelFormat) != VmbErrorSuccess) {
+        std::cerr << "Failed to get frame data!" << std::endl;
         return;
     }
 
+    // 2. 检查数据有效性
+    if (!pBuffer || width == 0 || height == 0) {
+        std::cerr << "Invalid frame data: width=" << width << ", height=" << height << std::endl;
+        return;
+    }
+
+    // 3. 更新V4L2设备格式（确保支持当前分辨率/PixelFormat）
     if (!UpdateVideoFormat(width, height, pixelFormat)) {
+        std::cerr << "Failed to update video format!" << std::endl;
         return;
     }
 
-    ssize_t written = write(m_videoFd, pBuffer, nSize);
-    if (written != (ssize_t)nSize) {
-        perror("Write to video device");
+    // 4. 处理Bayer RGGB格式（转换为YUYV）
+    if (pixelFormat == VmbPixelFormatBayerRG8) {
+        // 4.1 分配缓冲区
+        std::vector<VmbUchar_t> rgbData(width * height * 3);  // RGB24: width * height * 3
+        std::vector<VmbUchar_t> yuyvData(width * height * 2); // YUYV: width * height * 2
+
+        // 4.2 Bayer RGGB -> RGB
+        ConvertBayerRGGBToRGB(pBuffer, rgbData.data(), width, height);
+
+        // 4.3 RGB -> YUYV
+        ConvertRGBToYUYV(rgbData.data(), yuyvData.data(), width, height);
+
+        // 4.4 写入V4L2设备
+        ssize_t written = write(m_videoFd, yuyvData.data(), yuyvData.size());
+        if (written != (ssize_t)yuyvData.size()) {
+            perror("Write YUYV to video device failed");
+        }
+    } 
+    // 5. 处理非Bayer格式（直接写入）
+    else {
+        ssize_t written = write(m_videoFd, pBuffer, nSize);
+        if (written != (ssize_t)nSize) {
+            perror("Write raw data to video device failed");
+        }
     }
 
+    // 6. 统计性能
     auto endTime = std::chrono::high_resolution_clock::now();
     double latency = std::chrono::duration<double, std::milli>(endTime - startTime).count();
     m_totalLatency.store(m_totalLatency.load() + latency);
     m_frameCount++;
 
+    // 7. 重新排队帧
     m_pCamera->QueueFrame(pFrame);
 }
-
 double FrameObserver::GetAverageLatency() const {
     return m_frameCount > 0 ? m_totalLatency.load() / m_frameCount : 0.0;
 }

Vimba_6_0/VimbaCPP/Examples/AsynchronousGrab/VirtualCameraColor/Source/FrameObserver.h

@@ -8,6 +8,7 @@
 #include <mutex>
 #include <chrono>
 #include <atomic>
+#include <vector>  // Add for buffer storage
 
 namespace AVT {
 namespace VmbAPI {
@@ -29,6 +30,11 @@ private:
     void CloseVideoDevice();
     bool UpdateVideoFormat(VmbUint32_t width, VmbUint32_t height, VmbPixelFormatType pixelFormat);
 
+    // Bayer to RGB conversion
+    void ConvertBayerRGGBToRGB(const VmbUchar_t* bayerData, VmbUchar_t* rgbData, VmbUint32_t width, VmbUint32_t height);
+    // RGB to YUYV conversion
+    void ConvertRGBToYUYV(const VmbUchar_t* rgbData, VmbUchar_t* yuyvData, VmbUint32_t width, VmbUint32_t height);
+
     int m_videoFd;
     struct v4l2_format m_vfmt;
     std::mutex m_deviceMutex;

rknn_usb_rtsp-master/src/usb_to_rtsp.cpp

@@ -18,243 +18,194 @@
 
 #include "mk_mediakit.h"
 
-typedef struct
-{
+typedef struct {
     MppEncoder *encoder;
     mk_media media;
     mk_pusher pusher;
     const char *push_url;
 
-    int video_type=264;
-
+    int video_type = 264;
     int push_rtsp_port;
     std::string push_path_first;
     std::string push_path_second;
-
-
 } rknn_app_context_t;
 
-void release_media(mk_media *ptr)
-{
-    if (ptr && *ptr)
-    {
+void release_media(mk_media *ptr) {
+    if (ptr && *ptr) {
         mk_media_release(*ptr);
         *ptr = NULL;
     }
 }
 
-void release_pusher(mk_pusher *ptr)
-{
-    if (ptr && *ptr)
-    {
+void release_pusher(mk_pusher *ptr) {
+    if (ptr && *ptr) {
         mk_pusher_release(*ptr);
         *ptr = NULL;
     }
 }
 
-void release_track(mk_track *ptr)
-{
-    if (ptr && *ptr)
-    {
+void release_track(mk_track *ptr) {
+    if (ptr && *ptr) {
         mk_track_unref(*ptr);
         *ptr = NULL;
     }
 }
 
-
-
-// 函数定义  
 int padToMultipleOf16(int number) {
-    // 如果number已经是16的倍数，则直接返回  
     if (number % 16 == 0) {
         return number;
     }
-    // 否则，计算需要添加的额外量（即16 - (number % 16)）  
-    // 这等价于找到比number大的最小16的倍数，并减去number  
     int extra = 16 - (number % 16);
-    // 返回扩充后的数  
     return number + extra;
 }
 
-
-void API_CALL on_mk_push_event_func(void *user_data, int err_code, const char *err_msg)
-{
+void API_CALL on_mk_push_event_func(void *user_data, int err_code, const char *err_msg) {
     rknn_app_context_t *ctx = (rknn_app_context_t *)user_data;
-    if (err_code == 0)
-    {
-        // push success
+    if (err_code == 0) {
         log_info("push %s success!", ctx->push_url);
         printf("push %s success!\n", ctx->push_url);
-    }
-    else
-    {
+    } else {
         log_warn("push %s failed:%d %s", ctx->push_url, err_code, err_msg);
         printf("push %s failed:%d %s\n", ctx->push_url, err_code, err_msg);
         release_pusher(&(ctx->pusher));
     }
 }
 
-void API_CALL on_mk_media_source_regist_func(void *user_data, mk_media_source sender, int regist)
-{
+void API_CALL on_mk_media_source_regist_func(void *user_data, mk_media_source sender, int regist) {
     rknn_app_context_t *ctx = (rknn_app_context_t *)user_data;
     const char *schema = mk_media_source_get_schema(sender);
-    if (strncmp(schema, ctx->push_url, strlen(schema)) == 0)
-    {
+    if (strncmp(schema, ctx->push_url, strlen(schema)) == 0) {
         release_pusher(&(ctx->pusher));
-        if (regist)
-        {
+        if (regist) {
             ctx->pusher = mk_pusher_create_src(sender);
             mk_pusher_set_on_result(ctx->pusher, on_mk_push_event_func, ctx);
             mk_pusher_set_on_shutdown(ctx->pusher, on_mk_push_event_func, ctx);
             log_info("push started!");
             printf("push started!\n");
-        }
-        else
-        {
+        } else {
             log_info("push stoped!");
             printf("push stoped!\n");
         }
         printf("push_url:%s\n", ctx->push_url);
-    }
-    else
-    {
+    } else {
         printf("unknown schema:%s\n", schema);
     }
 }
 
-void API_CALL on_mk_shutdown_func(void *user_data, int err_code, const char *err_msg, mk_track tracks[], int track_count)
-{
-    printf("play interrupted: %d %s", err_code, err_msg);
-}
-
-int process_video_file(rknn_app_context_t *ctx, const char *video_path)
-{
-    // 读取视频
-    cv::VideoCapture cap(video_path);
-    if (!cap.isOpened())
-    {
-        printf("Failed to open video file: %s", video_path);
+int process_camera_stream(rknn_app_context_t *ctx) {
+    // Open camera device
+    cv::VideoCapture cap("/dev/video61");
+    if (!cap.isOpened()) {
+        printf("Failed to open camera device: /dev/video61\n");
+        return -1;
     }
-    // 使用前需要使用v4l2-ctl --device=/dev/video0 --list-formats-ext检查一下设备支持范围
-    cap.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G'));
-    // set width
-    cap.set(cv::CAP_PROP_FRAME_WIDTH, 1920);
-    // set height
-    cap.set(cv::CAP_PROP_FRAME_HEIGHT, 1080);
-    // set fps
-    cap.set(cv::CAP_PROP_FPS,30);
-    // 获取视频尺寸、帧率
+
+    // Set camera parameters based on v4l2-ctl output
+    cap.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('Y', 'U', 'Y', 'V'));
+    cap.set(cv::CAP_PROP_FRAME_WIDTH, 2452);
+    cap.set(cv::CAP_PROP_FRAME_HEIGHT, 2056);
+    cap.set(cv::CAP_PROP_FPS, 30);
+
+    // Get actual parameters
     int cap_width = cap.get(cv::CAP_PROP_FRAME_WIDTH);
     int cap_height = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
     int fps = cap.get(cv::CAP_PROP_FPS);
     
-    ctx->push_url = "rtsp://localhost/live/stream";
+    printf("Camera opened with resolution: %dx%d, FPS: %d\n", cap_width, cap_height, fps);
 
-    ctx->media = mk_media_create("__defaultVhost__", ctx->push_path_first.c_str(), ctx->push_path_second.c_str(), 0, 0, 0);
+    // RTSP stream setup
+    ctx->push_url = "rtsp://localhost/live/stream";
+    ctx->media = mk_media_create("__defaultVhost__", ctx->push_path_first.c_str(), 
+                                ctx->push_path_second.c_str(), 0, 0, 0);
 
     codec_args v_args = {0};
     mk_track v_track = mk_track_create(MKCodecH264, &v_args);
     mk_media_init_track(ctx->media, v_track);
-
     mk_media_init_complete(ctx->media);
     mk_media_set_on_regist(ctx->media, on_mk_media_source_regist_func, ctx);
 
-    // 初始化编码器
+    // Initialize encoder
     MppEncoder *mpp_encoder = new MppEncoder();
     MppEncoderParams enc_params;
     memset(&enc_params, 0, sizeof(MppEncoderParams));
     enc_params.width = cap_width;
     enc_params.height = cap_height;
-    enc_params.fmt = MPP_FMT_YUV420SP;
+    enc_params.fmt = MPP_FMT_YUV420SP;  // We'll convert from YUYV to YUV420SP
     enc_params.type = MPP_VIDEO_CodingAVC;
     mpp_encoder->Init(enc_params, ctx);
     ctx->encoder = mpp_encoder;
 
-    // mpp编码配置
+    // Variables for frame processing
     void *mpp_frame = NULL;
     int mpp_frame_fd = 0;
     void *mpp_frame_addr = NULL;
     int enc_data_size;
+    cv::Mat frame, yuv_frame;
 
-    int frame_index = 0;
-    int ret = 0;
-
-    // 画面
-    cv::Mat img;
-
-    while (true)
-    {
-        // 读取视频帧
-        cap >> img;
-        if (img.empty())
-        {
-            printf("Video end.");
+    while (true) {
+        // Read frame from camera
+        if (!cap.read(frame)) {
+            printf("Failed to read frame from camera\n");
             break;
         }
 
-        frame_index++;
-        // 结束计时
-        auto end_time = std::chrono::high_resolution_clock::now();
-        // 将当前时间点转换为毫秒级别的时间戳
-        auto millis = std::chrono::time_point_cast<std::chrono::milliseconds>(end_time).time_since_epoch().count();
+        if (frame.empty()) {
+            printf("Empty frame received\n");
+            continue;
+        }
 
-        // 获取解码后的帧
+        // Get current timestamp
+        auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(
+            std::chrono::system_clock::now().time_since_epoch()).count();
+
+        // Get encoder input buffer
         mpp_frame = ctx->encoder->GetInputFrameBuffer();
-        // 获取解码后的帧fd
         mpp_frame_fd = ctx->encoder->GetInputFrameBufferFd(mpp_frame);
-        // 获取解码后的帧地址
         mpp_frame_addr = ctx->encoder->GetInputFrameBufferAddr(mpp_frame);
 
-        rga_buffer_t src = wrapbuffer_fd(mpp_frame_fd, cap_width, cap_height, RK_FORMAT_YCbCr_420_SP,padToMultipleOf16(cap_width),padToMultipleOf16(cap_height));
+        // Convert YUYV to YUV420SP (NV12) using RGA
+        rga_buffer_t src = wrapbuffer_virtualaddr((void *)frame.data, 
+                                                 cap_width, cap_height, 
+                                                 RK_FORMAT_YUYV_422);
+        rga_buffer_t dst = wrapbuffer_fd(mpp_frame_fd, 
+                                       cap_width, cap_height, 
+                                       RK_FORMAT_YCbCr_420_SP,
+                                       padToMultipleOf16(cap_width),
+                                       padToMultipleOf16(cap_height));
 
+        // Perform color space conversion
+        imcvtcolor(src, dst, RK_FORMAT_YUYV_422, RK_FORMAT_YCbCr_420_SP);
+
+        // Encode frame
         int enc_buf_size = ctx->encoder->GetFrameSize();
-
         char *enc_data = (char *)malloc(enc_buf_size);
-
-        rga_buffer_t rgb_img = wrapbuffer_virtualaddr((void *)img.data, cap_width, cap_height, RK_FORMAT_BGR_888);
-        // 将RGB图像复制到src中
-        imcopy(rgb_img, src);
-
-        if (frame_index == 1)
-        {
-            enc_data_size = ctx->encoder->GetHeader(enc_data, enc_buf_size);
-        }
-        // 内存初始化
         memset(enc_data, 0, enc_buf_size);
         
         enc_data_size = ctx->encoder->Encode(mpp_frame, enc_data, enc_buf_size);
 
-        ret = mk_media_input_h264(ctx->media, enc_data, enc_data_size, millis, millis);
-        if (ret != 1)
-        {
+        // Push to RTSP
+        int ret = mk_media_input_h264(ctx->media, enc_data, enc_data_size, millis, millis);
+        if (ret != 1) {
             printf("mk_media_input_frame failed\n");
         }
-        if (enc_data != nullptr)
-        {
+
+        if (enc_data != nullptr) {
             free(enc_data);
         }
+
+        // Small delay to maintain frame rate
+        usleep(1000000 / fps);
     }
-    // 释放资源
+
+    // Release resources
     cap.release();
     release_track(&v_track);
     release_media(&ctx->media);
-
+    return 0;
 }
 
-int main(int argc, char **argv)
-{
-    int status = 0;
-    int ret;
-
-    if (argc != 2)
-    {
-        printf("Usage: %s<video_path>\n", argv[0]);
-        return -1;
-    }
-    char *stream_url = argv[1];               // 视频流地址
-    int video_type = 264;
-
-    // 初始化流媒体
+int main(int argc, char **argv) {
+    // Initialize media kit
     mk_config config;
     memset(&config, 0, sizeof(mk_config));
     config.log_mask = LOG_CONSOLE;
@@ -262,19 +213,20 @@ int main(int argc, char **argv)
     mk_env_init(&config);
     mk_rtsp_server_start(3554, 0);
 
-    rknn_app_context_t app_ctx;                      // 创建上下文
-    memset(&app_ctx, 0, sizeof(rknn_app_context_t)); // 初始化上下文
-    app_ctx.video_type = video_type;
-    app_ctx.push_path_first = "yunyan-live";
-    app_ctx.push_path_second = "test";
+    // Initialize application context
+    rknn_app_context_t app_ctx;
+    memset(&app_ctx, 0, sizeof(rknn_app_context_t));
+    app_ctx.video_type = 264;  // H.264
+    app_ctx.push_path_first = "live";
+    app_ctx.push_path_second = "stream";
 
-    process_video_file(&app_ctx, stream_url);
+    // Start processing camera stream
+    process_camera_stream(&app_ctx);
 
-    printf("waiting finish\n");
+    printf("Streaming finished\n");
     usleep(3 * 1000 * 1000);
 
-    if (app_ctx.encoder != nullptr)
-    {
+    if (app_ctx.encoder != nullptr) {
         delete (app_ctx.encoder);
         app_ctx.encoder = nullptr;
     }