OpenCV

From IPRE Wiki
Jump to: navigation, search

You can use OpenCV with your IPRE robot and pyro/myro. It has been tested on Mac OS 10.4 with the latest CVS versions of Myro and OpenCV. Once installed you can use the OpenCV Python Interface pretty quickly, however its interface is not quite ready for the CS-1 student. Some works needs to be done to wrap the opencv library for introductory students (e.g. startFindingFaces(), findFaces(), doneFindingFaces())

You can either use OpenCV to process images from:

Your webcam:

 capture = cvCreateCameraCapture(0)
 frame = cvQueryFrame(capture)

From a file:

 capture = cvCreateFileCapture('test.jpg')
 frame = cvQueryFrame(capture)

Myro

The Fluke's camera:

 frame = opencv.PIL2Ipl(takePicture().image.convert("RGB"))

You will need the latest version of myro for this above line of code to work. The way PIL is imported was mismatched between Myro and OpenCV which caused trouble in PIL2Ipl.

Visual Joystick

Visualjoystick.jpg

The mac can't use the gamepad because of the pygame/zelle interference on mac, but macs do have iSight cameras! And they are really easy to interface with opencv. I made few demos for controlling the robot using the camera Wii style. Here is one for controlling the robot using a visual joystick. We split the image up into for quadrants - joystick() style. Then use a pink ball or any other colored object to control the robot. If you move the ball up the robot goes forward, to the right the robot turns right etc.

This demo is based on the camshiftdemo.c file that comes with OpenCV and uses the camshift algorithm. You first need to drag a box around the object you are using to track and then move the object around to drive the robot. The 3 sliders are the parameters to the blob tracker (which track in HSV rather than RGB space). You might need to play around with those for best tracking performance. 'c' clears the current selection, and 'b' shows the current tracker in black and white.

##
## Based on camshiftdemo.c
##

import sys
from opencv import cv
from opencv import highgui
import opencv

from myro import *

go = False

image = None
hsv = None
hue = None
mask = None
backproject = None
ist = None

backproject_mode = 0
select_object = 0
track_object = 0

origin = cv.CvPoint()
selection = cv.CvRect()
track_window = cv.CvRect()
track_box = cv.CvBox2D()
track_comp = cv.CvConnectedComp()

hdims = 16
hranges = [[0, 180]]
vmin = 60
vmax = 256
smin = 65

def on_mouse(event, x, y, flags, param):

    global select_object, selection, image, origin, select_object, track_object

    if image is None:
        return

    if image.origin:
        y = image.height - y

    if select_object:
        selection.x = min(x,origin.x)
        selection.y = min(y,origin.y)
        selection.width = selection.x + cv.CV_IABS(x - origin.x)
        selection.height = selection.y + cv.CV_IABS(y - origin.y)
        
        selection.x = max( selection.x, 0 )
        selection.y = max( selection.y, 0 )
        selection.width = min( selection.width, image.width )
        selection.height = min( selection.height, image.height )
        selection.width -= selection.x
        selection.height -= selection.y

    if event == highgui.CV_EVENT_LBUTTONDOWN:
        origin = cv.cvPoint(x,y)
        selection = cv.cvRect(x,y,0,0)
        select_object = 1
    elif event == highgui.CV_EVENT_LBUTTONUP:
        select_object = 0
        if( selection.width > 0 and selection.height > 0 ):
            track_object = -1

def out_values():
    print "vmin =", vmin, "vmax =", vmax, "smin =", smin

def set_vmin(value):
    global vmin
    vmin = value
    out_values()

def set_vmax(value):
    global vmax
    vmax = value
    out_values()
    
def set_smin(value):
    global smin
    smin = value
    out_values()


def hsv2rgb (hue):
    # convert the hue value to the corresponding rgb value

    sector_data = [[0, 2, 1],
                   [1, 2, 0],
                   [1, 0, 2],
                   [2, 0, 1],
                   [2, 1, 0],
                   [0, 1, 2]]
    hue *= 0.1 / 3
    sector = cv.cvFloor (hue)
    p = cv.cvRound (255 * (hue - sector))
    if sector & 1:
        p ^= 255

    rgb = {}
    rgb [sector_data [sector][0]] = 255
    rgb [sector_data [sector][1]] = 0
    rgb [sector_data [sector][2]] = p

    return cv.cvScalar (rgb [2], rgb [1], rgb [0], 0)


if __name__ == '__main__':

    # use the webcam
    capture = highgui.cvCreateCameraCapture (0)

    # check that capture device is OK
    if not capture:
        print "Error opening capture device"
        sys.exit (1)
        
    # display a small howto use it
    print  "Hot keys: \n"
    print "\tESC - quit the program\n"
    print "\tc - stop the tracking\n"
    print "\tb - switch to/from backprojection view\n"
    print "To initialize tracking, select the object with mouse\n"

    # first, create the necessary windows
    highgui.cvNamedWindow ('VisualJoystick', highgui.CV_WINDOW_AUTOSIZE)

    # register the mouse callback
    highgui.cvSetMouseCallback ('VisualJoystick', on_mouse, None)
    
    highgui.cvCreateTrackbar( "Vmin", "VisualJoystick", vmin, 256, set_vmin)
    highgui.cvCreateTrackbar( "Vmax", "VisualJoystick", vmax, 256, set_vmax)
    highgui.cvCreateTrackbar( "Smin", "VisualJoystick", smin, 256, set_smin)


    if go:
        init()
        print getBattery()

    while True:

        frame = highgui.cvQueryFrame (capture)
            
        if frame is None:
            # no image captured... end the processing
            break

        if image is None:
            # create the images we need
            image = cv.cvCreateImage (cv.cvGetSize (frame), 8, 3)
            image.origin = frame.origin            
            hsv = cv.cvCreateImage( cv.cvGetSize(frame), 8, 3 )
            hue = cv.cvCreateImage( cv.cvGetSize(frame), 8, 1 )
            mask = cv.cvCreateImage( cv.cvGetSize(frame), 8, 1 )
            backproject = cv.cvCreateImage( cv.cvGetSize(frame), 8, 1 )
            hist = cv.cvCreateHist( [hdims], cv.CV_HIST_ARRAY, hranges, 1 )

        # flip the image
        cv.cvFlip (frame, image, 1)
        
        cv.cvCvtColor( image, hsv, cv.CV_BGR2HSV)

        cv.cvLine(image, cv.cvPoint(0, image.height/2), cv.cvPoint(image.width, image.height/2),
                  cv.CV_RGB(0,255,0), 2, 8, 0 )
        
        cv.cvLine(image, cv.cvPoint(image.width/2, 0), cv.cvPoint(image.width/2, image.height),
                  cv.CV_RGB(0,255,0), 2, 8, 0 )
        
        if track_object:
            _vmin = vmin
            _vmax = vmax

            cv.cvInRangeS( hsv,
                           cv.cvScalar(  0, smin,min(_vmin,_vmax),0),
                           cv.cvScalar(180, 256, max(_vmin,_vmax),0),
                           mask );

            cv.cvSplit( hsv, hue, None, None, None)

            if track_object < 0:
                max_val = 0.0                
                subhue = cv.cvGetSubRect(hue, selection)
                submask = cv.cvGetSubRect(mask, selection)
                cv.cvCalcHist( subhue, hist, 0, submask )
                
                # extract the min and max value of the histogram
                min_val, max_val, min_idx, max_idx = cv.cvGetMinMaxHistValue (hist)
                
                if (max_val):
                    cv.cvConvertScale( hist.bins, hist.bins, 255.0 / max_val, 0)
                else:
                    cv.cvConvertScale( hist.bins, hist.bins, 0.0, 0 )

                track_window = selection
                track_object = 1


            cv.cvCalcArrBackProject( hue, backproject, hist )
            
            cv.cvAnd( backproject, mask, backproject, 0 )
            cv.cvCamShift( backproject, track_window,
                           cv.cvTermCriteria( cv.CV_TERMCRIT_EPS | cv.CV_TERMCRIT_ITER, 10, 1 ),
                           track_comp, track_box )
            track_window = track_comp.rect
            
            if backproject_mode:
                cv.cvCvtColor( backproject, image, cv.CV_GRAY2BGR )
            if not image.origin:
                track_box.angle = -track_box.angle

            cv.cvEllipseBox(image, track_box, cv.CV_RGB(255,0,0), 3, cv.CV_AA, 0)
            
            if (track_box.size.width > 10 or track_box.size.height  > 10):

                rotate = ( (image.width/2.0) - track_box.center.x) / (image.width/2.0)
                translate = ((image.height/2.0) - track_box.center.y) / (image.height/2.0)
                
                #print "rotate =", rotate, "translate =", translate
                
                if go:
                    move(translate, rotate)
        
        if select_object and selection.width > 0 and selection.height > 0:
            subimg = cv.cvGetSubRect(image, selection)
            cv.cvXorS( subimage, cv.cvScalarAll(255), subimage, 0 )

        highgui.cvShowImage( "VisualJoystick", image )

        c = highgui.cvWaitKey(10)
        if c == '\x1b':
            break        
        elif c == 'b':
            backproject_mode ^= 1
        elif c == 'c':
            track_object = 0
            cv.cvZero( histimg )
            if go:
                stop()

if go:
    stop()

Face Detector

This demo used the fluke's camera to identify faces! When it spots a face it will speak "Hello Human"

OpenCVFace.jpg

#!/usr/bin/python
"""
This program is demonstration for face and object detection using haar-like features.
The program finds faces in a camera image or video stream and displays a red box around them.

Original C implementation by:  ?
Python implementation by: Roman Stanchak
Modified by Keith O'Hara to use Fluke's camera
"""
import sys
import opencv
from opencv.cv import *
from opencv.highgui import *

from myro import *
import PIL

# Global Variables
cascade = None
storage = cvCreateMemStorage(0)
cascade_name = "../../data/haarcascades/haarcascade_frontalface_alt.xml"

# Parameters for haar detection
# From the API:
# The default parameters (scale_factor=1.1, min_neighbors=3, flags=0) are tuned 
# for accurate yet slow object detection. For a faster operation on real video 
# images the settings are: 
# scale_factor=1.2, min_neighbors=2, flags=CV_HAAR_DO_CANNY_PRUNING, 
# min_size=<minimum possible face size
min_size = cvSize(20,20)
image_scale = 1.3
haar_scale = 1.2
min_neighbors = 2
haar_flags = 0


def detect_and_draw( img ):
    # allocate temporary images
    gray = cvCreateImage( cvSize(img.width,img.height), 8, 1 );
    small_img = cvCreateImage( cvSize( cvRound (img.width/image_scale),
						               cvRound (img.height/image_scale)), 8, 1 );

    # convert color input image to grayscale
    cvCvtColor( img, gray, CV_BGR2GRAY );

    # scale input image for faster processing
    cvResize( gray, small_img, CV_INTER_LINEAR );

    cvEqualizeHist( small_img, small_img );
    
    cvClearMemStorage( storage );

    if( cascade ):
        t = cvGetTickCount();
        faces = cvHaarDetectObjects( small_img, cascade, storage,
                                     haar_scale, min_neighbors, haar_flags, min_size );
        t = cvGetTickCount() - t;
        print "detection time = %gms" % (t/(cvGetTickFrequency()*1000.));
        if faces:
            for face_rect in faces:

                speak("Hello human", 0)
                
                # the input to cvHaarDetectObjects was resized, so scale the 
                # bounding box of each face and convert it to two CvPoints
                pt1 = cvPoint( int(face_rect.x*image_scale), int(face_rect.y*image_scale))
                pt2 = cvPoint( int((face_rect.x+face_rect.width)*image_scale),
                               int((face_rect.y+face_rect.height)*image_scale) )
                cvRectangle( img, pt1, pt2, CV_RGB(255,0,0), 3, 8, 0 );

    cvShowImage( "result", img );


if __name__ == '__main__':

    if len(sys.argv) > 1:

        if sys.argv[1].startswith("--cascade="):
            cascade_name = sys.argv[1][ len("--cascade="): ]
            if len(sys.argv) > 2:
                input_name = sys.argv[2]

        elif sys.argv[1] == "--help" or sys.argv[1] == "-h":
            print "Usage: facedetect --cascade=\"<cascade_path>\" [filename|camera_index]\n" ;
            sys.exit(-1)

    
    # the OpenCV API says this function is obsolete, but we can't
    # cast the output of cvLoad to a HaarClassifierCascade, so use this anyways
    # the size parameter is ignored
    cascade = cvLoadHaarClassifierCascade( cascade_name, cvSize(1,1) );
    
    if not cascade:
        print "ERROR: Could not load classifier cascade"
        sys.exit(-1)
    
    cvNamedWindow( "result", 1 );

    init()
    
    frame_copy = None
    while True: 

        p = takePicture()
        frame = opencv.PIL2Ipl(p.image.convert("RGB"))
        
        if( not frame ):
            break;
        if( not frame_copy ):
            frame_copy = cvCreateImage( cvSize(frame.width,frame.height),
                                        IPL_DEPTH_8U, frame.nChannels );
        if( frame.origin == IPL_ORIGIN_TL ):
            cvCopy( frame, frame_copy );
        else:
            cvFlip( frame, frame_copy, 0 );
            
        detect_and_draw( frame_copy );
        
        if( cvWaitKey( 10 ) >= 0 ):
            break;

Pyro

Here is a sample Pyro Device that uses an OpenCV Camera and processes it to find humans.

You can use like:

pyrobot -rTest -dOpenCVCamera

The program will ask for a OpenCV Camera index.

# file: OpenCVCamera.py
""" Needs an opencv camera. """

from pyrobot.system.share import ask
from pyrobot.camera.fake import ManualFakeCamera
import opencv
from opencv.cv import *
from opencv.highgui import *

# Global Variables
cascade = None
storage = cvCreateMemStorage(0)
cascade_name = "/usr/share/opencv/haarcascades/haarcascade_frontalface_alt.xml"

# Parameters for haar detection
# From the API:
# The default parameters (scale_factor=1.1, min_neighbors=3, flags=0) are tuned 
# for accurate yet slow object detection. For a faster operation on real video 
# images the settings are: 
# scale_factor=1.2, min_neighbors=2, flags=CV_HAAR_DO_CANNY_PRUNING, 
# min_size=<minimum possible face size
min_size = cvSize(20,20)
image_scale = 1.3
haar_scale = 1.2
min_neighbors = 2
haar_flags = 0

class OpenCVCamera(ManualFakeCamera):
    def __init__(self, ocv_index):
        ## Setup OpenCV camera:
        self.ocv_index = ocv_index
        self.cascade = cvLoadHaarClassifierCascade( cascade_name, cvSize(1,1) );
        if not self.cascade:
            print "ERROR: Could not load classifier cascade"
        self.capture = cvCreateCameraCapture(self.ocv_index)
        frame = opencv.highgui.cvQueryFrame(self.capture)
        self.frame_copy = None
        width = frame.width
        height = frame.height
        ManualFakeCamera.__init__(self, width, height, 3)

    def update(self):
        frame = opencv.highgui.cvQueryFrame(self.capture)
        if(not frame): return
        if(not self.frame_copy):
            self.frame_copy = cvCreateImage( cvSize(frame.width,frame.height),
                                             IPL_DEPTH_8U, frame.nChannels )
        if(frame.origin == IPL_ORIGIN_TL ):
            cvCopy( frame, self.frame_copy )
        else:
            cvFlip( frame, self.frame_copy, 0 )
        rects = self.detect(self.frame_copy )

    def detect(self,  img):
        # allocate temporary images
        gray = cvCreateImage( cvSize(img.width,img.height), 8, 1 );
        small_img = cvCreateImage( cvSize( cvRound (img.width/image_scale),
                                           cvRound (img.height/image_scale)), 8, 1 )
        # convert color input image to grayscale
        cvCvtColor( img, gray, CV_BGR2GRAY )
        # scale input image for faster processing
        cvResize( gray, small_img, CV_INTER_LINEAR )
        cvEqualizeHist( small_img, small_img )
        cvClearMemStorage( storage )
        if( self.cascade ):
            t = cvGetTickCount();
            faces = cvHaarDetectObjects( small_img, self.cascade, storage,
                                         haar_scale, min_neighbors, haar_flags, min_size );
            t = cvGetTickCount() - t;
            print "detection time = %gms" % (t/(cvGetTickFrequency()*1000.));
            retval = []
            if faces:
                for face_rect in faces:
                    print "Hello human"
                    # the input to cvHaarDetectObjects was resized, so scale the 
                    # bounding box of each face and convert it to two CvPoints
                    #pt1 = cvPoint( int(face_rect.x*image_scale), int(face_rect.y*image_scale))
                    #pt2 = cvPoint( int((face_rect.x+face_rect.width)*image_scale),
                    #               int((face_rect.y+face_rect.height)*image_scale) )
                    retval.append( (face_rect.x, face_rect.y, face_rect.width, face_rect.y))
                    #cvRectangle( img, pt1, pt2, CV_RGB(255,0,0), 3, 8, 0 );
        #cvShowImage( "result", img );
        return retval
    
def INIT(robot):
    retval = ask("Please enter the OpenCV index number of the camera",
                 (("OpenCV Camera index", "0"),
                  ))
    ocv_index = int(retval["OpenCV Camera index"])
    if retval["ok"]:
        return {"camera": OpenCVCamera(ocv_index)}
    else:
        raise "Cancelled!"