Multi tool use

QVideoFrame::map() crashes

I am trying to get images from the camera using QCamera.
I thought I should derive from QAbstractVideoSurface and implement present(), where a QVideoFrame, representing the current image captured by the camera, is sent as a parameter.

QAbstractVideoSurface

present()

QVideoFrame

As I need to do some processing, I tried to map() my frame, get the data with bits(), do whatever I have to do, then unmap() it. However I have a crash on map()

map()

bits()

unmap()

map()

Here are the errors I get :

W libTest.so: (null):0 ((null)): Unsupported viewfinder pixel format
D SensorManager: registerListener :: 6, LSM6DSL Acceleration Sensor, 200000, 0,
E libEGL : call to OpenGL ES API with no current context (logged once per thread)
E GLConsumer: [SurfaceTexture-0-546-0] attachToContext: invalid current EGLDisplay
F libc : Fatal signal 11 (SIGSEGV), code 1, fault addr 0x4 in tid 663 (qtMainLoopThrea)


What am I doing wrong ?

Here is a full application code :

///////////////////////////////////////////////
//main.cpp
///////////////////////////////////////////////

#include <QGuiApplication>
#include <QQmlApplicationEngine>
#include <QQmlContext>

#include "camera_engine.h"

int main(int argc, char *argv)
{
QCoreApplication::setAttribute(Qt::AA_EnableHighDpiScaling);

QGuiApplication app(argc, argv);
QQmlApplicationEngine engine;

CameraEngine camEngine;
engine.rootContext()->setContextProperty("cameraEngine", &camEngine);

engine.load(QUrl(QStringLiteral("qrc:/main.qml")));
if (engine.rootObjects().isEmpty())
return -1;

return app.exec();
}


///////////////////////////////////////////////
//camera_engine.h
///////////////////////////////////////////////

#ifndef __CAMERA_ENGINE_H__
#define __CAMERA_ENGINE_H__

#include <QCamera>

#include "image_reader.h"

class CameraEngine : public QObject {
Q_OBJECT
public:
explicit CameraEngine(QCamera::Position pos = QCamera::BackFace);
~CameraEngine();

public slots:
void start();
private:
QCamera mCamera;
ImageReader mImageReader;
};

#endif // __CAMERA_ENGINE_H__


///////////////////////////////////////////////
//camera_engine.cpp
///////////////////////////////////////////////

#include "camera_engine.h"

CameraEngine::CameraEngine(QCamera::Position pos) : mCamera(pos)
{
mCamera.setViewfinder(&mImageReader);

QCameraViewfinderSettings viewFinderSettings;
viewFinderSettings.setResolution(640, 480);
viewFinderSettings.setMinimumFrameRate(30);
viewFinderSettings.setMaximumFrameRate(30);
viewFinderSettings.setPixelFormat(QVideoFrame::Format_RGB24);
mCamera.setViewfinderSettings(viewFinderSettings);
}

CameraEngine::~CameraEngine()
{
if (mCamera.state() == QCamera::ActiveState) {
mCamera.stop();
}
}

void CameraEngine::start()
{
mCamera.start();
}


///////////////////////////////////////////////
//image_reader.h
///////////////////////////////////////////////

#ifndef CAMERA_IMAGE_READER_H
#define CAMERA_IMAGE_READER_H

#include <QAbstractVideoSurface>

class ImageReader : public QAbstractVideoSurface {
Q_OBJECT
public:
ImageReader() = default;
~ImageReader() = default;

virtual bool present(const QVideoFrame& frame);
virtual QList<QVideoFrame::PixelFormat> supportedPixelFormats(QAbstractVideoBuffer::HandleType type) const;
};

#endif // CAMERA_IMAGE_READER_H


///////////////////////////////////////////////
//image_reader.cpp
///////////////////////////////////////////////

#include "image_reader.h"
#include <QDebug>

bool ImageReader::present(const QVideoFrame &frame)
{
QVideoFrame currentFrame = frame;

currentFrame.map(QAbstractVideoBuffer::ReadOnly); //crashes here
// Do something
currentFrame.unmap();

return true;
}

QList<QVideoFrame::PixelFormat> ImageReader::supportedPixelFormats(QAbstractVideoBuffer::HandleType type) const
{
Q_UNUSED(type)
return QList<QVideoFrame::PixelFormat>() << QVideoFrame::Format_RGB24;
}

///////////////////////////////////////////////
//main.qml
///////////////////////////////////////////////

import QtQuick 2.11
import QtQuick.Controls 2.2

ApplicationWindow {
id: window

visible: true
width: 640
height: 480

Component.onCompleted: cameraEngine.start()
}


Edit: So, I think this might be because my QVideoFrame is stored as an OpenGL texture, and my present() function might not be running on the OpenGL thread, thus not finding the OpenGL ES Context.
Is there a way to make sure it's running on the right thread ?

present()

Edit2: I found this : http://doc.qt.io/qt-5/qtquick-scenegraph-openglunderqml-example.html

Maybe I can adapt this code to get a valid OpenGL ES context. Unfortunately I have no time to do it now. I'll try that Monday, unless someone have a better suggestion, and tell you the results.

Edit3: So, apparently my solution was not the good one, I get a crash (SIGSEGV) on initializeOpenGLFunctions();

initializeOpenGLFunctions();

I saw Antonio Dias' answer, using a VideoOutput with the function grabToImage, tried it, and it seemed to work, but, if I understand correctly, grabToImage "draws" the VideoOutput in CPU memory, and I lose some metadata I was planning to get with QMediaMetaData in the process.

VideoOutput

grabToImage

grabToImage

VideoOutput

QMediaMetaData

I also tried to use the NDK directly, but the camera requires an API level of at least 24, and even after setting all relevant settings I found, it does not seem to use it.

Edit4: I actually do not know what I did, but my program ended up using the right API level, so I'll go with the NDK solution for now.

2 Answers
2

...unless someone have a better suggestion...

My simple way to capture images from Camera object in QML is calling grabToImage and pass the image to C++ to process it!

Camera

grabToImage

This method does not oblige you to process all the frames of the active camera, you can grab only when you are ready or desire it!

Keep in mind, especially in android, that the VideoOutput object needs to be visible in order to grab anything, you can't, for example, grab images with screen turned off on Android. However you not need to fill the screen, you only have to gave some area of the screen to the VideoOutput and keep it visible and on top.

VideoOutput

VideoOutput

Also, note that reduce the size of VideoOutput does not reduce the maximum size you can capture, for example you can capture a 1280x720 image with a VideoOutput with smaller size.

VideoOutput

VideoOutput

Also the aspect ratio must be preserved when resizing VideoOutput to keep the aspect ratio of the image.

VideoOutput

Code:

You can see full source here!

main.cpp:

#include <QGuiApplication>
#include <QQmlApplicationEngine>
#include "camerahelper.h"

int main(int argc, char *argv)
{
#if defined(Q_OS_WIN)
QCoreApplication::setAttribute(Qt::AA_EnableHighDpiScaling);
#endif

QGuiApplication app(argc, argv);

QQmlApplicationEngine engine;
engine.load(QUrl(QStringLiteral("qrc:/main.qml")));
if (engine.rootObjects().isEmpty())
return -1;

CameraHelper camera_helper;
camera_helper.start(engine.rootObjects().first());

return app.exec();
}


camerahelper.h:

#ifndef CAMERAHELPER_H
#define CAMERAHELPER_H

#include <QtCore>
#include <QtGui>
#include <QtQuick>

class CameraHelper : public QObject
{
Q_OBJECT
public:
explicit CameraHelper(QObject *parent = nullptr);

signals:

public slots:
void start(QObject *qml_obj);
void grab();

private slots:
void frameReady(const QVariant &frame_variant);

private:
QObject *m_qml_obj;
};

#endif // CAMERAHELPER_H


camerahelper.cpp:

#include "camerahelper.h"

CameraHelper::CameraHelper(QObject *parent) : QObject(parent)
{

}

void CameraHelper::start(QObject *qml_obj)
{
m_qml_obj = qml_obj;

//Connect the QML frameReady SIGNAL to our frameReady SLOT
connect(m_qml_obj, SIGNAL(frameReady(QVariant)), this, SLOT(frameReady(QVariant)));

//Do the first grab
grab();
}

void CameraHelper::grab()
{
//Size of the captured image
QSize size = QSize(320, 240);

//Pass grab size to QML and wait for captured image on the frameReady SIGNAL
QMetaObject::invokeMethod(m_qml_obj, "grab", Q_ARG(QVariant, size.width()), Q_ARG(QVariant, size.height()));
}

void CameraHelper::frameReady(const QVariant &frame_variant)
{
QQuickItemGrabResult *grab_result = qvariant_cast<QQuickItemGrabResult*>(frame_variant); //Cast from QVariant
QImage frame = grab_result->image(); //Get the QImage
grab_result->deleteLater(); //Release QQuickItemGrabResult

//Depending on OS the image can have different formats,
//use convertToFormat to unify all possibles formats to one
frame = frame.convertToFormat(QImage::Format_RGB32);

//Frame is ready to use

grab(); //Do the next frame grab
}


main.qml:

import QtQuick 2.10
import QtQuick.Window 2.10
import QtMultimedia 5.8

Window
{
visible: true
color: "black"
width: 640
height: 480
title: qsTr("Hello World")

signal frameReady(var frame)

Camera
{
id: camera
viewfinder.resolution: "320x240"
}

VideoOutput
{
id: videoOutput
source: camera
autoOrientation: true
visible: true
anchors.fill: parent
}

function grab(grab_width, grab_height)
{
if (!visible)
return

videoOutput.grabToImage(function(frame)
{
frameReady(frame) //Emit frameReady SIGNAL
}, Qt.size(grab_width, grab_height))
}
}


I finally managed to make it work. Apparently QCamera has some problems on Android, so I had to use a QML Camera, and display it with a VideoOutput, and apply a filter through which I can get my images.

QCamera

Camera

VideoOutput

This filter is made in two parts : one deriving from QAbstractVideoFilter, the other deriving from QVideoFilterRunnable.

QAbstractVideoFilter

QVideoFilterRunnable

Here is the code :

////////////////////////////////////////////////////////////////////
// main.cpp
////////////////////////////////////////////////////////////////////

#include <QGuiApplication>
#include <QQmlApplicationEngine>

#include "myfilter.hpp"

int main(int argc, char *argv)
{
QCoreApplication::setAttribute(Qt::AA_EnableHighDpiScaling);

QGuiApplication app(argc, argv);
QQmlApplicationEngine engine;

qmlRegisterType<MyFilter>("example.myfilter", 1, 0, "MyFilter");

engine.load(QUrl(QStringLiteral("qrc:/main.qml")));
if (engine.rootObjects().isEmpty())
return -1;

return app.exec();
}


////////////////////////////////////////////////////////////////////
// myfilter.hpp
////////////////////////////////////////////////////////////////////

#ifndef MYFILTER_H
#define MYFILTER_H

#include <QAbstractVideoFilter>

class MyFilterRunnable : public QVideoFilterRunnable {
public:
QVideoFrame run(QVideoFrame *input, const QVideoSurfaceFormat &surfaceFormat, RunFlags flags);
};


class MyFilter : public QAbstractVideoFilter
{
public:
QVideoFilterRunnable* createFilterRunnable();
};

#endif // MYFILTER_H

////////////////////////////////////////////////////////////////////
// myfilter.cpp
////////////////////////////////////////////////////////////////////

#include "myfilter.hpp"

#include <QOpenGLContext>
#include <QOpenGLFunctions>

QVideoFrame MyFilterRunnable::run(QVideoFrame *input, const QVideoSurfaceFormat &surfaceFormat, QVideoFilterRunnable::RunFlags flags)
{
QImage img(input->width(), input->height(), QImage::Format_RGBA8888);
bool success = false;
if (input->handleType() == QAbstractVideoBuffer::GLTextureHandle) {
GLuint textureId = input->handle().toUInt();
QOpenGLContext *ctx = QOpenGLContext::currentContext();
QOpenGLFunctions *f = ctx->functions();
GLuint fbo;
f->glGenFramebuffers(1, &fbo);
GLuint prevFbo;
f->glGetIntegerv(GL_FRAMEBUFFER_BINDING, (GLint *) &prevFbo);
f->glBindFramebuffer(GL_FRAMEBUFFER, fbo);
f->glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, textureId, 0);
f->glReadPixels(0, 0, input->width(), input->height(), GL_RGBA, GL_UNSIGNED_BYTE, img.bits());
f->glBindFramebuffer(GL_FRAMEBUFFER, prevFbo);
success = true;
} // else handle other types

if( success ) {
// Process image
return QVideoFrame(img);
} else {
return *input; //Could not apply filter, return unmodified input
}
}

QVideoFilterRunnable *MyFilter::createFilterRunnable()
{
return new MyFilterRunnable;
}


////////////////////////////////////////////////////////////////////
// main.qml
////////////////////////////////////////////////////////////////////

import QtQuick 2.11
import QtQuick.Controls 2.2
import QtMultimedia 5.9
import example.myfilter 1.0

ApplicationWindow {
id: window

visible: true
width: 640
height: 480

Camera {
id: camera
}
MyFilter {
id: filter
}
VideoOutput {
source: camera
autoOrientation: true
filters: [ filter ]
anchors.fill: parent
}
}


(My run implementation is adapted from here : http://code.qt.io/cgit/qt/qtmultimedia.git/tree/examples/multimedia/video/qmlvideofilter_opencl/rgbframehelper.h)

run

By clicking "Post Your Answer", you acknowledge that you have read our updated terms of service, privacy policy and cookie policy, and that your continued use of the website is subject to these policies.