1
\$\begingroup\$

These two classes are part of a larger program that I am developing. The user is supposed to interact with the program using handheld red, blue, green or yellow discs and moving them in front of a webcam. The program tracks the movement and color of these discs and draws small circles of the corresponding color in a certain area of the program's window. The program also calculates how much of each color the user has "painted" in that area.

The standard OpenCV formula is used. Grab a frame from webcam, convert to HSV format, use the HSV thresholds for the colored discs to make 4 binary images, erode and dilate to get rid of noise, use HoughCircles to detect the circles in the binary images, so on and so forth...

While the code works and I am able to use the user's input in the other parts of the program, the color detection itself is very slow/laggy and hence the tracking and drawing is slow as well. Looking at the raw camera feed, it's almost running at ~10fps or lower. The two relevant classes are:

AirPainter.cpp

#include "AirPainter.h"
#define POSITIVE(n) ((n) < 0 ? 0 - (n) : (n))


//default constructor
AirPainter::AirPainter() :AirPainter(352, 240) {}

AirPainter::AirPainter(int frameWidth, int frameHeigth)
{
    FRAME_WIDTH = frameWidth;
    FRAME_HEIGHT = frameHeigth;
    MAX_OBJECT_AREA = FRAME_HEIGHT*FRAME_WIDTH / 3;

    capture.open(0);
    if (!capture.isOpened())
    {
        cout << "capture is not openned";
    }

    //set height and width of capture frame
    capture.set(CV_CAP_PROP_FRAME_WIDTH, FRAME_WIDTH);
    capture.set(CV_CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT);
    capture.read(cameraFeed);

    //create empty mat with black background
    drawingCanvas = Mat::zeros(cameraFeed.size(), CV_8UC3);

    //create object to store area drawn by the user
    Object oriRed("red"), oriBlue("blue"), oriGreen("green"), oriYellow("yellow");

    allColor.push_back(oriRed);
    allColor.push_back(oriBlue);
    allColor.push_back(oriGreen);
    allColor.push_back(oriYellow);

    ignore = false;
}

//convert int to string
string intToString(int number) {

    std::stringstream ss;
    ss << number;
    return ss.str();
}

//set to only able to draw within a certain radius of circle
bool AirPainter::drawable(int x, int y)
{
    int centerx = 470;
    int centery = 362;
    int hypo = (int)sqrt(pow(POSITIVE(centerx - x), 2) + pow(POSITIVE(centery - y), 2));
    if (hypo > 284)
    {
        return false;
    }
    return true;
}

//draw the line on the canvas
void AirPainter::drawLine(vector<Object> theObjects, Mat &drawingCanvasTemp)
{
    for (int i = 0; i < theObjects.size(); i++) {
        if (drawable(theObjects.at(i).getXPos(), theObjects.at(i).getYPos()) && ignore)
        {
            line(
                drawingCanvas,
                Point(theObjects.at(i).getXPos(), theObjects.at(i).getYPos()),
                Point(theObjects.at(i).getXPos(), theObjects.at(i).getYPos()),
                theObjects.at(i).getColor(),
                70
            );
        }
        else
        {
            line(
                drawingCanvasTemp,
                Point(theObjects.at(i).getXPos(), theObjects.at(i).getYPos()),
                Point(theObjects.at(i).getXPos(), theObjects.at(i).getYPos()),
                theObjects.at(i).getColor(),
                70
            );
        }
    }
}

//filter noise
void AirPainter::morphOps(Mat &thresh) {

    //create structuring element that will be used to "dilate" and "erode" image.
    //the element chosen here is a 3px by 3px rectangle
    Mat erodeElement = getStructuringElement(MORPH_RECT, Size(3, 3));
    //dilate with larger element so make sure object is nicely visible
    Mat dilateElement = getStructuringElement(MORPH_RECT, Size(8, 8));

    erode(thresh, thresh, erodeElement);
    erode(thresh, thresh, erodeElement);

    dilate(thresh, thresh, dilateElement);
    dilate(thresh, thresh, dilateElement);
}


//only track circle object
void AirPainter::TrackCircle(Object color, Mat &drawingCanvasTemp)
{
    vector <Object> objects;
    bool objectFound = false;
    vector<Vec3f> circles;

    // Apply the Hough Transform to find the circles
    HoughCircles(threshold, circles, CV_HOUGH_GRADIENT, 1, threshold.rows / 8, 100, 20, 40, 0);//threshold, circles, CV_HOUGH_GRADIENT, 1, threshold.rows / 8, 100, 20, 40, 0
    for (size_t i = 0; i < circles.size(); i++)
    {
        Point center(cvRound(circles[i][0]), cvRound(circles[i][1]));
        int radius = cvRound(circles[i][2]);
        Object object;
        object.setXPos(cvRound(circles[i][0]));
        object.setYPos(cvRound(circles[i][1]));
        object.setType(color.getType());
        object.setColor(color.getColor());
        objects.push_back(object);
        objectFound = true;

        // circle center
        circle(cameraFeed, center, 3, Scalar(0, 255, 0), -10, 8, 0);
        // circle outline
        circle(cameraFeed, center, radius, Scalar(0, 0, 255), 3, 8, 0);
    }
    if (objectFound)
    {
        drawLine(objects, drawingCanvasTemp);
    }
}


//managing specific color to be filter and drawn
void AirPainter::ColorManager(Mat &drawingCanvasTemp, Object colorObject)
{
    cvtColor(cameraFeed, HSV, COLOR_BGR2HSV);
    inRange(HSV, colorObject.getHSVmin(), colorObject.getHSVmax(), threshold);
    morphOps(threshold);
    GaussianBlur(threshold, threshold, Size(9, 9), 2, 2);

    imshow(colorObject.getType(), threshold);

}

//get the area where the user had drawn
void AirPainter::ColorArea(Object &color)
{
    Mat tmpHsv;
    Mat tmpThreshold;

    cvtColor(drawingCanvas, tmpHsv, COLOR_BGR2HSV);

    inRange(tmpHsv, color.getHSVminArea(), color.getHSVmaxArea(), tmpThreshold);
    morphOps(tmpThreshold);
    Moments moment = moments(tmpThreshold);

    color.setArea((moment.m00 / 10000) / 8290 * 100);
}

//get the area that the user had drawn
string AirPainter::getColorArea(string name)
{
    int area = 0;
    for (int i = 0; i < allColor.size(); i++)
    {
        if (allColor.at(i).getType() == name)
        {
            area = allColor.at(i).getArea();
        }
    }
    return intToString(area);
}

//reset canvas into blank and area information to 0
void AirPainter::resetArea_Canvas()
{
    for (int i = 0; i < allColor.size(); i++)
    {
        allColor.at(i).setArea(0);
    }
    drawingCanvas = Mat::zeros(cameraFeed.size(), CV_8UC3);
}

vector<Object> AirPainter::getAllColor()
{
    return allColor;
}


void AirPainter::run()
{
    capture >> cameraFeed;

    if (!cameraFeed.data)
    {
        cout << "error taking video" << endl;
        return;
    }
    else
    {
        Mat drawingCanvasTemp = Mat::zeros(cameraFeed.size(), CV_8UC3);

        flip(cameraFeed, cameraFeed, 1);
        src = cameraFeed;

        //convert frame from BGR to HSV colorspace
        medianBlur(cameraFeed, cameraFeed, 3);
        cvtColor(cameraFeed, HSV, COLOR_BGR2HSV);

        Object blue("blue"), yellow("yellow"), red("red"), green("green");

        ColorManager(drawingCanvasTemp, yellow);
        ColorManager(drawingCanvasTemp, green);
        ColorManager(drawingCanvasTemp, blue);
        ColorManager(drawingCanvasTemp, red);


        //search for area drawn by the user
        for (int i = 0; i < allColor.size(); i++)
        {
            ColorArea(allColor[i]);
        }

        drawingCanvasTemp = drawingCanvas + drawingCanvasTemp;
        showCanvas = drawingCanvasTemp;

        imshow("Original Image", cameraFeed);

    }

}

Object.cpp

#include "Object.h"

Object::Object()
{
    //set values for default constructor
    setType("Object");
    setColor(Scalar(0, 0, 0));

}

Object::Object(string name) {

    setType(name);

    if (name == "blue") {

        setHSVminArea(Scalar(91, 213, 145));
        setHSVmaxArea(Scalar(143, 255, 255));

        setHSVmin(Scalar(88, 84, 51));
        setHSVmax(Scalar(119, 183, 198));

        //BGR value for Blue:
        setColor(Scalar(250, 0, 0));

    }
    if (name == "green") {

        setHSVminArea(Scalar(47, 24, 162));
        setHSVmaxArea(Scalar(100, 255, 255));

        setHSVmin(Scalar(37, 79, 47));
        setHSVmax(Scalar(85, 173, 255));

        //BGR value for Green:
        setColor(Scalar(0, 250, 0));

    }
    if (name == "yellow") {

        setHSVminArea(Scalar(20, 124, 123));
        setHSVmaxArea(Scalar(30, 255, 255));

        setHSVmin(Scalar(20, 96, 147));
        setHSVmax(Scalar(60, 152, 255));

        //BGR value for yellow:
        setColor(Scalar(0, 255, 255));

    }
    if (name == "red") {

        setHSVminArea(Scalar(0, 100, 100));
        setHSVmaxArea(Scalar(10, 255, 255));

        setHSVmin(Scalar(150, 148, 126));
        setHSVmax(Scalar(179, 255, 255));

        //BGR value for Red:
        setColor(Scalar(0, 0, 250));

    }
}

Object::~Object(void)
{
}

int Object::getXPos() {

    return Object::xPos;

}

void Object::wait()
{
    waitKey(30);
}

void Object::setXPos(int x) {

    Object::xPos = x;

}

int Object::getYPos() {

    return Object::yPos;

}

void Object::setYPos(int y) {

    Object::yPos = y;

}

Scalar Object::getHSVmin() {

    return Object::HSVmin;

}
Scalar Object::getHSVmax() {

    return Object::HSVmax;
}

Scalar Object::getHSVminArea() {

    return Object::HSVminArea;

}
Scalar Object::getHSVmaxArea() {

    return Object::HSVmaxArea;
}

void Object::setHSVmin(Scalar min) {

    Object::HSVmin = min;
}


void Object::setHSVmax(Scalar max) {

    Object::HSVmax = max;
}

void Object::setHSVminArea(Scalar min) {

    Object::HSVminArea = min;
}


void Object::setHSVmaxArea(Scalar max) {

    Object::HSVmaxArea = max;
}

void Object::setArea(double Area)
{
    Object::area = Area;
}

double Object::getArea()
{
    return Object::area;
}
\$\endgroup\$
5
  • \$\begingroup\$ Did you profile the code? I'm pretty sure the Hough transform is the resource hog, in which case a code review won't help you. There are likely better methods to extract shapes, so I'd ask stackoverflow or dsp.se for a better algorithm (like the famous cola can question). \$\endgroup\$ – Rainer P. Mar 26 '18 at 22:16
  • \$\begingroup\$ @RainerP. I'm not sure how to do code profiling, it's certainly something I could look at. Kind of a silly question, but couldn't I comment out the HoughCircles function to see whether the camera frame-rate improves? Because I did, obviously there is no circle detection anymore, but there is still the same lag. Not sure if this an indication of anything... \$\endgroup\$ – ArmenHeat Mar 26 '18 at 23:26
  • \$\begingroup\$ Can you make this into a complete and self-contained program? I know it's still going to be hard for reviewers to run it without a suitable input file, but it might help. I see some small performance hogs (such as calculating hypo the slow way many times rather than using std::hypot()). \$\endgroup\$ – Toby Speight Mar 27 '18 at 8:59
  • \$\begingroup\$ It would certainly be useful to see the class definitions for Object and AirPainter, at least. \$\endgroup\$ – Toby Speight Mar 27 '18 at 9:17
  • \$\begingroup\$ Is std::to_string faster than using the manual approach via stringstream? \$\endgroup\$ – yuri Mar 27 '18 at 11:15

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.