Converting ffmpeg AVFrame Yuv420P to OpenCV RGB cv::Mat

Question

Thanks to Rotem and his help with this issue, I can offer an implementation that allows to use cvtColor to convert an AVFrame YUV420P into a BGR cv::Mat. The key is to create a temporary intermediate cv::Mat where all the AVFrame data is copied in the way cvtColor expects it.

#include <cstdint>
#include <vector>
#include <opencv2/opencv.hpp>
#include <opencv2/imgproc.hpp>
extern "C" {
#include <libavutil/frame.h>
}

class AVFrameYuvToBgrMatConverter {
private:
    mutable cv::Mat m_yuv;
    ///We use a support buffer because in this way we can force opencv to use the same stride of AVFrame,
    ///so that the y channel, which is 2 thirds of the data, can be copied into the intermediate matrix in one go
    mutable std::vector<uint8_t> m_yuv_buffer;

public:
    void convert(const AVFrame& frame, cv::Mat& dest) const {
        const auto width = frame.width;
        const auto height = frame.height;
        ///Check the input has an even number of cols, and rows divisible by 4
        assert((width & 0x1) + (height & 0x11) == 0);
        const std::size_t y_stride = frame.linesize[0];
        const auto y_src = frame.data[0];
        {
            const std::size_t new_size = y_stride * 3 / 2 * height;
            if (new_size > m_yuv_buffer.size()) {
                ///We resize the buffer only if it needs to grow
                m_yuv_buffer.resize(new_size);
            }
        }
        ///The internal cv::Mat m_yuv doesn't allocate a buffer, so it is efficient, but yet creating a cv::Mat
        ///has some overhead so we only do that when necessary
        if (m_yuv.cols != width || m_yuv.rows != height || m_yuv.step1() != y_stride ||
            m_yuv.ptr() != m_yuv_buffer.data()) {
            ///Unfortunately cv::Mat::create does not support passing a preallocated buffer
            m_yuv = cv::Mat(height * 3 / 2, width, CV_8UC1, m_yuv_buffer.data(), y_stride);
        }
        ///Copy y channel
        memcpy(m_yuv.ptr(), y_src, y_stride * height);

        ///Copy uv channels
        const auto u_stride = frame.linesize[1];
        const uint8_t* u_src = frame.data[1];
        const auto v_stride = frame.linesize[2];
        const uint8_t* v_src = frame.data[2];
        if (static_cast<int>(y_stride) == width && u_stride == width / 2 && v_stride == width / 2) {
            ///In this case we can copy the u and v channels in one go
            memcpy(m_yuv.ptr(height), u_src, width * height / 4);
            memcpy(m_yuv.ptr(height * 5 / 4), v_src, width * height / 4);
        } else {
            ///In this case we have to proceed line by line
            auto dst = m_yuv.ptr(height);
            for (int i = 0; i < height / 4; i++) {
                memcpy(dst, u_src, width / 2);
                u_src += u_stride;
                memcpy(dst + width / 2, u_src, width / 2);
                u_src += u_stride;
                dst += y_stride;
            }
            for (int i = 0; i < height / 4; i++) {
                memcpy(dst, v_src, width / 2);
                v_src += v_stride;
                memcpy(dst + width / 2, v_src, width / 2);
                v_src += v_stride;
                dst += y_stride;
            }
        }
        ///cvtColor input is finally ready
        cv::cvtColor(m_yuv, dest, cv::COLOR_YUV2BGR_I420, 3);
    }
};

Ideally one could go inside OpenCV code, see how the conversion is done, and adapt it to use directly AVFrame's buffers without having to reshuffle bytes beforehand.

The alternative to perform this conversion is to use ffmpeg, this is spelled out as part of this answer. It would be eventually worth to investigate which one is the most efficient, I'll post the results if I ever get to do that.