OpenCV Eye Tracking

Nov 2, 2008 | Tags: OpenCV | del.icio.us del.icio.us | digg Digg

This is a simple program that displays live video from a webcam and tracks user's eye. The system tracks user's eye with a given template, which was manually selected using mouse.

When the user initially clicks the eye feature, a box is drawn around the square and the subimage within this square is cropped out of the image frame. The cropped image is used as a template to find the position of the feature in subsequent frames. The system determines the position of the feature using Sum of Square Differences (SQD) method. To reduce extensive computation, the system tracks the feature in a "search window", a small area around the position of the feature in previous frame.

Listing 1: OpenCV Eye Tracking

  1. /**
  2.  * Display video from webcam and track user's eye with
  3.  * manually selected template.
  4.  *
  5.  * Author    Nash <me [at] nashruddin.com>
  6.  * License   GPL
  7.  * Website   http://nashruddin.com
  8.  */
  9.  
  10. #include <stdio.h>
  11. #include "cv.h"
  12. #include "highgui.h"
  13.  
  14. #define  TPL_WIDTH       12      /* template width       */
  15. #define  TPL_HEIGHT      12      /* template height      */
  16. #define  WINDOW_WIDTH    24      /* search window width  */
  17. #define  WINDOW_HEIGHT   24      /* search window height */
  18. #define  THRESHOLD       0.3
  19.  
  20. IplImage *frame, *tpl, *tm;
  21. int      object_x0, object_y0, is_tracking = 0;
  22.  
  23. void mouseHandler( int event, int x, int y, int flags, void *param );
  24. void trackObject();
  25.  
  26. /* main code */
  27. int main( int argc, char** argv )
  28. {
  29.     CvCapture   *capture;
  30.     int         key;
  31.  
  32.     /* initialize camera */
  33.     capture = cvCaptureFromCAM( 0 );
  34.  
  35.     /* always check */
  36.     if( !capture ) return 1;
  37.  
  38.     /* get video properties, needed by template image */
  39.     frame = cvQueryFrame( capture );
  40.     if ( !frame ) return 1;
  41.    
  42.     /* create template image */
  43.     tpl = cvCreateImage( cvSize( TPL_WIDTH, TPL_HEIGHT ),
  44.                          frame->depth, frame->nChannels );
  45.    
  46.     /* create image for template matching result */
  47.     tm = cvCreateImage( cvSize( WINDOW_WIDTH  - TPL_WIDTH  + 1,
  48.                                 WINDOW_HEIGHT - TPL_HEIGHT + 1 ),
  49.                         IPL_DEPTH_32F, 1 );
  50.    
  51.     /* create a window and install mouse handler */
  52.     cvNamedWindow( "video", CV_WINDOW_AUTOSIZE );
  53.     cvSetMouseCallback( "video", mouseHandler, NULL );
  54.    
  55.     while( key != 'q' ) {
  56.         /* get a frame */
  57.         frame = cvQueryFrame( capture );
  58.  
  59.         /* always check */
  60.         if( !frame ) break;
  61.  
  62.         /* 'fix' frame */
  63.         cvFlip( frame, frame, -1 );
  64.         frame->origin = 0;
  65.        
  66.         /* perform tracking if template is available */
  67.         if( is_tracking ) trackObject();
  68.        
  69.         /* display frame */
  70.         cvShowImage( "video", frame );
  71.  
  72.         /* exit if user press 'q' */
  73.         key = cvWaitKey( 1 );
  74.     }
  75.  
  76.     /* free memory */
  77.     cvDestroyWindow( "video" );
  78.     cvReleaseCapture( &capture );
  79.     cvReleaseImage( &tpl );
  80.     cvReleaseImage( &tm );
  81.    
  82.     return 0;
  83. }
  84.  
  85. /* mouse handler */
  86. void mouseHandler( int event, int x, int y, int flags, void *param )
  87. {
  88.     /* user clicked the image, save subimage as template */
  89.     if( event == CV_EVENT_LBUTTONUP ) {
  90.         object_x0 = x - ( TPL_WIDTH  / 2 );
  91.         object_y0 = y - ( TPL_HEIGHT / 2 );
  92.        
  93.         cvSetImageROI( frame,
  94.                        cvRect( object_x0,
  95.                                object_y0,
  96.                                TPL_WIDTH,
  97.                                TPL_HEIGHT ) );
  98.         cvCopy( frame, tpl, NULL );
  99.         cvResetImageROI( frame );
  100.  
  101.         /* template is available, start tracking! */
  102.         fprintf( stdout, "Template selected. Start tracking... \n" );
  103.         is_tracking = 1;
  104.     }
  105. }
  106.  
  107. /* track object */
  108. void trackObject()
  109. {
  110.     CvPoint minloc, maxloc;
  111.     double  minval, maxval;
  112.  
  113.     /* setup position of search window */
  114.     int win_x0 = object_x0 - ( ( WINDOW_WIDTH  - TPL_WIDTH  ) / 2 );
  115.     int win_y0 = object_y0 - ( ( WINDOW_HEIGHT - TPL_HEIGHT ) / 2 );
  116.    
  117.     /*
  118.      * Ooops, some bugs here.
  119.      * If the search window exceed the frame boundaries,
  120.      * it will trigger errors.
  121.      *
  122.      * Add some code to make sure that the search window
  123.      * is still within the frame.
  124.      */
  125.    
  126.     /* search object in search window */
  127.     cvSetImageROI( frame,
  128.                    cvRect( win_x0,
  129.                            win_y0,
  130.                            WINDOW_WIDTH,
  131.                            WINDOW_HEIGHT ) );
  132.     cvMatchTemplate( frame, tpl, tm, CV_TM_SQDIFF_NORMED );
  133.     cvMinMaxLoc( tm, &minval, &maxval, &minloc, &maxloc, 0 );
  134.     cvResetImageROI( frame );
  135.    
  136.     /* if object found... */
  137.     if( minval <= THRESHOLD ) {
  138.         /* save object's current location */
  139.         object_x0 = win_x0 + minloc.x;
  140.         object_y0 = win_y0 + minloc.y;
  141.  
  142.         /* and draw a box there */
  143.         cvRectangle( frame,
  144.                      cvPoint( object_x0, object_y0 ),
  145.                      cvPoint( object_x0 + TPL_WIDTH,
  146.                               object_y0 + TPL_HEIGHT ),
  147.                      cvScalar( 0, 0, 255, 0 ), 1, 0, 0 );
  148.     } else {
  149.         /* if not found... */
  150.         fprintf( stdout, "Lost object.\n" );
  151.         is_tracking = 0;
  152.     }
  153. }
  154.  

Notice line 117-124, you should add some code to make sure that the search window is still within the frame. If the search window exceed the frame boundaries, it will trigger errors. I will leave this as an exercise for you, dear reader.

Some features to add for the next version:

  • Automatic eye detection
  • Save the video to file
  • Head movement for User Interface

Send suggestion, bug reports or any comments about this program to me [at] nashruddin.com.

Related Articles

Recommended Books

book cover Learning OpenCV: Computer Vision with the OpenCV Library
By: Gary Bradski, Adrian Kaehler
This book is the "de facto" OpenCV User's Manual. It provides a practical, pragmatic, accessible book on computer vision, with algorithmic explanation and concrete example code snippets. Written by the creators of OpenCV, no doubt you should obtain a copy.

book cover Machine Vision: Theory, Algorithms, Practicalities (Signal Processing and its Applications)
By: E. R. Davies
This book provides a solid and concrete foundation to computer vision from engineering point of view. Use Learning OpenCV from Gary Bradsky to learn how to use OpenCV, and use this book to understand how OpenCV works behind the screen.

The Downloads

31 Comments

Stephen Smith on Dec 14, 2008:

Having trouble compiling, i emailed you. (Windows)

Risa Indah on Jan 5, 2009:

I currently try to implement an eye-detection using openCV, it shall be used in an openSource HCI-inteface that allows to control the mouse.i want to use haar cascade methode, but i am stil beginer in opencv.
could you tell me step by step to create a simple example for haarcascade methode and how to create xml file such as haarcascade_frontalface_alt.xml on C:\Program Files\OpenCV\data\haarcascades ?
i would very much apprechiate some working code examples ...
thanks...

Nash on Jan 6, 2009:

From the OpenCV Reference Manual:

"First, a classifier (namely a cascade of boosted classifiers working with haar-like features) is trained with a few hundreds of sample views of a particular object (i.e., a face or a car), called positive examples, that are scaled to the same size (say, 20x20), and negative examples - arbitrary images of the same size."

"After a classifier is trained, it can be applied to a region of interest (of the same size as used during the training) in an input image. The classifier outputs a '1' if the region is likely to show the object (i.e., face/car), and '0' otherwise."

And here's How-to build a cascade of boosted classifiers based on Haar-like features.

However, I would suggest you to use simpler method to locate human eyes e.g: blink detection.

vanathy on Jan 7, 2009:

very useful

Pernnie on Jan 8, 2009:

I am a beginner using OpenCV. I have problems with compiling a program. I changed the directory, where I installed OpenCV. I go to command (I am using Windows), to directory where I copied all files and write make, but the error appears. 'gcc' is not recognized as an internal or external command, operable program or batch file. **error 1** deleting main.o
Thank you very much...

Nash on Jan 8, 2009:

You need to install MinGW first. It's a free C compiler from GNU.

When you've installed MinGW, you can run gcc to compile C programs. Feel free to contact me if you encounter problems.

Fernando(Brazil) on Jan 12, 2009:

Nash, thanks for your code.Is very useful to me. Compiled using MS Visual Studio 2008 and despite some error worked well. I plan to improve its application and place to move the mouse. You already put something in your code?
Thanks

Nash on Jan 12, 2009:

Hi Fernando,
I'm trying to add automatic eye detection and Human Computer Interaction (HCI) to the code.

Andres on May 26, 2009:

I am a beginner in OpenCV and C programming in general, and I am trying to come up with the code to make sure that the search window is still within the frame. (lines 117- 124) I was wondering if you could give me some hints or ideas of how to come up with it. Maybe I need to understand better how the search window works.

I would really appreciate your help.

Nash on May 27, 2009:

I leave that as a homework for you.

To understand the search window, why don't you draw the borders? Here it is.

cvRectangle(
  frame,
  cvPoint(win_x0, win_y0),
  cvPoint(win_x0 + WINDOW_WIDTH, win_y0 + WINDOW_HEIGHT),
  cvScalar( 0, 255, 0, 0), 1, 0, 0
);
put the code above at line 117 to view the search window. Good luck.

k09 on Jul 3, 2009:

Thankyou for very useful post. But I have some concern about this program.
First, I changed some codes to tracking eyes even after lose object. It worked well but when action moves fast, no tracking procedure is operated. That is because object_x0 and object_y0 unchange after fast moving.
Because you used correlation coefficient to detect eyes, how about using eyes classfier to detect eyes coordinates and then detect eyes with template. I tried it but it wasn't cool at all.

Otherwise, I am doing project about blink detection using correlation coefficient but the most important part - tracking eyes didn't work well. Can you give some advices about tracking eyes and your next version?

Thank you so much!

Nash on Jul 4, 2009:

Using eye classifier to detects eyes in live video is a bad idea since the result is very poor. There is a better method for finding eyes, by detecting blinks, which I'm currently try to implement.

Once the eyes has successfully located, we can extract the template and use it for tracking.

Btw, it seems like you have done the blink detection. Why don't you post your code in this forum so we can learn it from you?

Metal3d on Jul 5, 2009:

I compiled you example on Linux using:

gcc $(pkg-config --cflags --libs opencv) eye-detect.c -o eye-detect

I compiles without error and binay runs. A window appears and... nothing. No camera, nothing...

If I press 'q', window closes and no error appears, so the main loop runs.

I'm cheking what's happend

Nash on Jul 5, 2009:

Make sure your camera works on Linux.

k09 on Jul 7, 2009:

My source code is only modified stuffs from your codes to detect eyeblink when object is lost so there's is nothing new. If I have something new I will post it and ask you some advices.

Anyway, I have problem with any program that relates to mouse event in Fedora (all OpenCV examples etc,..). When I move mouse to openCV windows, the program is cancelled (no click, no press key) (for examle I run example 4-1 in Learning OpenCV book). Do you have the same problem before? Thank you so much for helping.

Metal3d on Jul 8, 2009:

Nash, yes it works and python implementation of opencv woks for me. I can see my cam output etc...

I have tried basic test to only see my cam... and same error.

I insists: python implentation of opencv works

fawzy on Jul 13, 2009:

hey , nash your post was very useful for me , I am working on a project like camera mouse to move computer mouse by user's eye , and implementing mouse click by using blink detection , but i have some problems with detecting eye blink so can u help me on that, thx in advance

Nash on Jul 14, 2009:

hello,
Currently I'm writing the code for blink detection and real-time eye tracking. I'll post it within the next few days.

Updated Jul 16, 2009:
Check this one out: Real Time Eye Tracking and Blink Detection.

Enrique on Aug 26, 2009:

Hello,
I'm a newbie in opencv. i compiled your facedetection.c code in your prev post but its not working. it show the mini window once then a message
"Assertion failed : casecade && storage && capture , file.........line 21"
how can i solve this.

Laxmi on Aug 27, 2009:

i want to download the header files of your code i.e cv.h and highgui.h

Nash on Aug 28, 2009:

@Enrique:

Generally it is because the code cannot find the xml file. Also make sure that your camera works by running the eye tracking above.

@Laxmi:

The header files are part of OpenCV library. Download and install it in your computer.

Max on Sep 13, 2009:

that's great!

but with two tracked objects it would be even better...

Michael on Oct 7, 2009:

@Metal3d: Would you like to share your Python code?

diana on Oct 25, 2010:

hye.. i try your code.. it's great.. tq so much... by the way, do you have idea how to track eye movement as mouse?

Janidu on Feb 22, 2011:

Hi Nash,

I modified your code to select the object to track. My program works for small objects but always throws me an error Sizes of input arguments do not match (output image should be (W - w + 1)x(H - h + 1)) , for other objects.

It would be great to get some help. Where can I post the code for you to see.

Cheers.

Simone Bonaria on Mar 24, 2011:

Hi, my video window is all grey , i can't see anything.
Can you help me?

rudi on Mar 31, 2011:

hi,
I have the same problem as Metal3d...but eye tracking with blink detection code compiles and shows the webcam feed.Although no tracking of eyes in that..Any suggestions??


Mdaud on May 25, 2011:

Hi Nash I run you code on ubuntu 11.04 1. opposite appearance of cam (upside down) I use hp pavilion dm1

Matt on Jun 1, 2011:

Hey guys,
I'm a noob with all of this. I don't know how to run this program. I used cmake to compile the script using the opencv scripts. Now where do I go from here? I have an allbuild solution file, install solution, etc. Sorry for my ignorance. Thanks

Igor Benko on Jun 4, 2011:

Hi nash,

this is one great example and it really helped me out. But one thing I don't understand is why you used cvFlip(frame, frame, -1) to 'fix' frame? I understand what this function does, but in my case it just turns picture upside down, I didn't find any other difference in functionality. Is this related to some old bug of OpenCV?

Thanks!

Cheers

Willy on Jul 7, 2011:

Is there any code for Live Face detection
???? please help....

Leave a comment

Name (required)
Email (will not be published) (required)
Website

Characters left = 1000

Tags

Correlation  HTTP  Image  Networking  OpenCV  PHP  Projects  Regex  SEO  Security  WebService  opencv 

Recent Posts

  1. What Do You Expect from a Quick OpenCV Guide?
  2. OpenCV Utility: Reading Image Pixels Value
  3. OpenCV Circular ROI
  4. OpenCV 2.0 Installation on Windows XP and Visual Studio 2008
  5. Runtime ROI Selection using Mouse
View Archives

About the Author

avatar Cool PHP programmer writing cool PHP scripts. Feel free to contact
ICQ 489571630
Skype dede_bl4ckheart
Yahoo dede_bl4ckheart
Google   nashruddin.amin

Recommended Sites:

OpenCV Code Examples
HTML and CSS Tutorials
Stop Dreaming Start Action
Online Quran and Translation