OpenCV Eye Tracking

Nov 2, 2008 | Tags: OpenCV | del.icio.us del.icio.us | digg Digg

This is a simple program that displays live video from a webcam and tracks user's eye. The system tracks user's eye with a given template, which was manually selected using mouse.

When the user initially clicks the eye feature, a box is drawn around the square and the subimage within this square is cropped out of the image frame. The cropped image is used as a template to find the position of the feature in subsequent frames. The system determines the position of the feature using Sum of Square Differences (SQD) method. To reduce extensive computation, the system tracks the feature in a "search window", a small area around the position of the feature in previous frame.

Listing 1: OpenCV Eye Tracking

  1. /**
  2.  * Display video from webcam and track user's eye with
  3.  * manually selected template.
  4.  *
  5.  * Author    Nash <me [at] nashruddin.com>
  6.  * License   GPL
  7.  * Website   http://nashruddin.com
  8.  */
  9.  
  10. #include <stdio.h>
  11. #include "cv.h"
  12. #include "highgui.h"
  13.  
  14. #define  TPL_WIDTH       12      /* template width       */
  15. #define  TPL_HEIGHT      12      /* template height      */
  16. #define  WINDOW_WIDTH    24      /* search window width  */
  17. #define  WINDOW_HEIGHT   24      /* search window height */
  18. #define  THRESHOLD       0.3
  19.  
  20. IplImage *frame, *tpl, *tm;
  21. int      object_x0, object_y0, is_tracking = 0;
  22.  
  23. void mouseHandler( int event, int x, int y, int flags, void *param );
  24. void trackObject();
  25.  
  26. /* main code */
  27. int main( int argc, char** argv )
  28. {
  29.     CvCapture   *capture;
  30.     int         key;
  31.  
  32.     /* initialize camera */
  33.     capture = cvCaptureFromCAM( 0 );
  34.  
  35.     /* always check */
  36.     if( !capture ) return 1;
  37.  
  38.     /* get video properties, needed by template image */
  39.     frame = cvQueryFrame( capture );
  40.     if ( !frame ) return 1;
  41.    
  42.     /* create template image */
  43.     tpl = cvCreateImage( cvSize( TPL_WIDTH, TPL_HEIGHT ),
  44.                          frame->depth, frame->nChannels );
  45.    
  46.     /* create image for template matching result */
  47.     tm = cvCreateImage( cvSize( WINDOW_WIDTH  - TPL_WIDTH  + 1,
  48.                                 WINDOW_HEIGHT - TPL_HEIGHT + 1 ),
  49.                         IPL_DEPTH_32F, 1 );
  50.    
  51.     /* create a window and install mouse handler */
  52.     cvNamedWindow( "video", CV_WINDOW_AUTOSIZE );
  53.     cvSetMouseCallback( "video", mouseHandler, NULL );
  54.    
  55.     while( key != 'q' ) {
  56.         /* get a frame */
  57.         frame = cvQueryFrame( capture );
  58.  
  59.         /* always check */
  60.         if( !frame ) break;
  61.  
  62.         /* 'fix' frame */
  63.         cvFlip( frame, frame, -1 );
  64.         frame->origin = 0;
  65.        
  66.         /* perform tracking if template is available */
  67.         if( is_tracking ) trackObject();
  68.        
  69.         /* display frame */
  70.         cvShowImage( "video", frame );
  71.  
  72.         /* exit if user press 'q' */
  73.         key = cvWaitKey( 1 );
  74.     }
  75.  
  76.     /* free memory */
  77.     cvDestroyWindow( "video" );
  78.     cvReleaseCapture( &capture );
  79.     cvReleaseImage( &tpl );
  80.     cvReleaseImage( &tm );
  81.    
  82.     return 0;
  83. }
  84.  
  85. /* mouse handler */
  86. void mouseHandler( int event, int x, int y, int flags, void *param )
  87. {
  88.     /* user clicked the image, save subimage as template */
  89.     if( event == CV_EVENT_LBUTTONUP ) {
  90.         object_x0 = x - ( TPL_WIDTH  / 2 );
  91.         object_y0 = y - ( TPL_HEIGHT / 2 );
  92.        
  93.         cvSetImageROI( frame,
  94.                        cvRect( object_x0,
  95.                                object_y0,
  96.                                TPL_WIDTH,
  97.                                TPL_HEIGHT ) );
  98.         cvCopy( frame, tpl, NULL );
  99.         cvResetImageROI( frame );
  100.  
  101.         /* template is available, start tracking! */
  102.         fprintf( stdout, "Template selected. Start tracking... \n" );
  103.         is_tracking = 1;
  104.     }
  105. }
  106.  
  107. /* track object */
  108. void trackObject()
  109. {
  110.     CvPoint minloc, maxloc;
  111.     double  minval, maxval;
  112.  
  113.     /* setup position of search window */
  114.     int win_x0 = object_x0 - ( ( WINDOW_WIDTH  - TPL_WIDTH  ) / 2 );
  115.     int win_y0 = object_y0 - ( ( WINDOW_HEIGHT - TPL_HEIGHT ) / 2 );
  116.    
  117.     /*
  118.      * Ooops, some bugs here.
  119.      * If the search window exceed the frame boundaries,
  120.      * it will trigger errors.
  121.      *
  122.      * Add some code to make sure that the search window
  123.      * is still within the frame.
  124.      */
  125.    
  126.     /* search object in search window */
  127.     cvSetImageROI( frame,
  128.                    cvRect( win_x0,
  129.                            win_y0,
  130.                            WINDOW_WIDTH,
  131.                            WINDOW_HEIGHT ) );
  132.     cvMatchTemplate( frame, tpl, tm, CV_TM_SQDIFF_NORMED );
  133.     cvMinMaxLoc( tm, &minval, &maxval, &minloc, &maxloc, 0 );
  134.     cvResetImageROI( frame );
  135.    
  136.     /* if object found... */
  137.     if( minval <= THRESHOLD ) {
  138.         /* save object's current location */
  139.         object_x0 = win_x0 + minloc.x;
  140.         object_y0 = win_y0 + minloc.y;
  141.  
  142.         /* and draw a box there */
  143.         cvRectangle( frame,
  144.                      cvPoint( object_x0, object_y0 ),
  145.                      cvPoint( object_x0 + TPL_WIDTH,
  146.                               object_y0 + TPL_HEIGHT ),
  147.                      cvScalar( 0, 0, 255, 0 ), 1, 0, 0 );
  148.     } else {
  149.         /* if not found... */
  150.         fprintf( stdout, "Lost object.\n" );
  151.         is_tracking = 0;
  152.     }
  153. }
  154.  

Notice line 117-124, you should add some code to make sure that the search window is still within the frame. If the search window exceed the frame boundaries, it will trigger errors. I will leave this as an exercise for you, dear reader.

Some features to add for the next version:

  • Automatic eye detection
  • Save the video to file
  • Head movement for User Interface

Send suggestion, bug reports or any comments about this program to me [at] nashruddin.com.

Related Articles

Recommended Book

book cover Learning OpenCV: Computer Vision with the OpenCV Library
By: Gary Bradski, Adrian Kaehler
This book is the "de facto" OpenCV User's Manual. It provides a practical, pragmatic, accessible book on computer vision, with algorithmic explanation and concrete example code snippets. Written by the creators of OpenCV, no doubt you should obtain a copy.

The Downloads

10 Comments

Stephen Smith on Dec 14, 2008:

Having trouble compiling, i emailed you. (Windows)

Risa Indah on Jan 5, 2009:

I currently try to implement an eye-detection using openCV, it shall be used in an openSource HCI-inteface that allows to control the mouse.i want to use haar cascade methode, but i am stil beginer in opencv.
could you tell me step by step to create a simple example for haarcascade methode and how to create xml file such as haarcascade_frontalface_alt.xml on C:\Program Files\OpenCV\data\haarcascades ?
i would very much apprechiate some working code examples ...
thanks...

Nash on Jan 6, 2009:

From the OpenCV Reference Manual:

"First, a classifier (namely a cascade of boosted classifiers working with haar-like features) is trained with a few hundreds of sample views of a particular object (i.e., a face or a car), called positive examples, that are scaled to the same size (say, 20x20), and negative examples - arbitrary images of the same size."

"After a classifier is trained, it can be applied to a region of interest (of the same size as used during the training) in an input image. The classifier outputs a '1' if the region is likely to show the object (i.e., face/car), and '0' otherwise."

And here's How-to build a cascade of boosted classifiers based on Haar-like features.

However, I would suggest you to use simpler method to locate human eyes e.g: blink detection.

vanathy on Jan 7, 2009:

very useful

Pernnie on Jan 8, 2009:

I am a beginner using OpenCV. I have problems with compiling a program. I changed the directory, where I installed OpenCV. I go to command (I am using Windows), to directory where I copied all files and write make, but the error appears. 'gcc' is not recognized as an internal or external command, operable program or batch file. **error 1** deleting main.o
Thank you very much...

Nash on Jan 8, 2009:

You need to install MinGW first. It's a free C compiler from GNU.

When you've installed MinGW, you can run gcc to compile C programs. Feel free to contact me if you encounter problems.

Fernando(Brazil) on Jan 12, 2009:

Nash, thanks for your code.Is very useful to me. Compiled using MS Visual Studio 2008 and despite some error worked well. I plan to improve its application and place to move the mouse. You already put something in your code?
Thanks

Nash on Jan 12, 2009:

Hi Fernando,
I'm trying to add automatic eye detection and Human Computer Interaction (HCI) to the code.

Andres on May 26, 2009:

I am a beginner in OpenCV and C programming in general, and I am trying to come up with the code to make sure that the search window is still within the frame. (lines 117- 124) I was wondering if you could give me some hints or ideas of how to come up with it. Maybe I need to understand better how the search window works.

I would really appreciate your help.

Nash on May 27, 2009:

I leave that as a homework for you.

To understand the search window, why don't you draw the borders? Here it is.

  1. cvRectangle(
  2.   frame,
  3.   cvPoint(win_x0, win_y0),
  4.   cvPoint(win_x0 + WINDOW_WIDTH, win_y0 + WINDOW_HEIGHT),
  5.   cvScalar( 0, 255, 0, 0), 1, 0, 0
  6. );
put the code above at line 117 to view the search window. Good luck.

Leave a comment

Name (required)
Email (will not be published) (required)
Website

Characters left = 1000

Tags

Correlation  HTTP  Image  Networking  OpenCV  PHP  Projects  Regex  SEO  Security  WebService 

Newsletter

Send me your new post on:
OpenCV
PHP
Projects 		Networking
Regex
to my email:
No, I won't spam your email.

Recent Posts

  1. PHP Script for Converting Relative to Absolute URL
  2. Streaming OpenCV Videos Over the Network
  3. Password Protected Images with PHP
  4. OpenCV Region of Interest (ROI)
  5. Web Based Surveillance System with OpenCV, PHP and Javascript

Popular Posts

  1. OpenCV Eye Detection
  2. OpenCV Examples Part 1
  3. OpenCV Face Detection
  4. OpenCV Eye Tracking
  5. Display Video from Webcam
  6. Phase Correlation in OpenCV
  7. Simple File Server and Downloader Script
  8. Building Dynamic SQL String from Associative Array
  9. Template Matching with OpenCV
  10. Fetching a Web Page From Your PHP Code

About the Author

avatar Cool PHP programmer writing cool PHP scripts. Feel free to contact
Tel. +62 31 8662872
+62 856 338 6017
ICQ 489571630
Skype dede_bl4ckheart
Yahoo dede_bl4ckheart
Google nashruddin.amin