null null labs | DirectShow.cpp

DirectShow.cpp

/*
*  DirectShow.cpp
*
*  Created by Daisuke Nogami on Nov, 05.
*  opyright 2005 Daisuke Nogami [null-null.net]. Some rights reserved.
*/
 
#include "StdAfx.h"
#include "DirectShow.h"
 
 
#include "math.h"
#include "time.h"
 
 
#include <GL/glut.h>
 
// texture size
#define TSZW 256
#define TSZH 256
 
// [flag] capture image reversed
#define REVERSE
 
 
DirectShow::DirectShow(){
  this->btmp = new unsigned char[TSZW*TSZH*3];
  this->_cHeight = TSZH;
  this->_cWidth = TSZW;
 
  // initialize directshow and opencv
  IGraphBuilder* pGraph = NULL;
  IMediaControl* pMC = NULL;
  ICaptureGraphBuilder2* pCapture = NULL;
  IBaseFilter* pF  = NULL;
  ISampleGrabber* pGrab = NULL;
 
  ICreateDevEnum* pDevEnum = NULL;
 
  init();
}
 
DirectShow::~DirectShow(){
  ///////
  delete []this->btmp;
  this->btmp=NULL;
}
 
void DirectShow::init(){
  // directshow
  CoInitialize(NULL);  
 
  // キャプチャデバイスを探す
  IBaseFilter  *pbf = NULL;
  IMoniker * pMoniker = NULL;
    ULONG cFetched;
    // デバイス列挙子を作成
    
    CoCreateInstance( CLSID_SystemDeviceEnum, NULL, CLSCTX_INPROC,
            IID_ICreateDevEnum, (void ** ) &pDevEnum);
 
    // ビデオキャプチャデバイス列挙子を作成
    IEnumMoniker * pClassEnum = NULL;
    pDevEnum -> CreateClassEnumerator(
          CLSID_VideoInputDeviceCategory, &pClassEnum, 0);
 
    if (pClassEnum == NULL){
    printf("ビデオキャプチャデバイスは存在しません\n");
    pDevEnum -> Release();
    CoUninitialize();
        return ;
    }
    // 最初に見つかったビデオキャプチャデバイスのオブジェクトの
  // インタフェースを得る
    pClassEnum -> Next(1, &pMoniker, &cFetched);
  pMoniker -> BindToObject( 0, 0, IID_IBaseFilter, (void**)&pbf );
   
  // ---- フィルタグラフの準備 ----
    // フィルタグラフを作り、インターフェースを得る
    CoCreateInstance( CLSID_FilterGraph, NULL, CLSCTX_INPROC,
            IID_IGraphBuilder, (void **) &pGraph);
    pGraph -> QueryInterface( IID_IMediaControl, (LPVOID *) &pMC );
  // キャプチャフィルタをフィルタグラフに追加
    pGraph -> AddFilter( pbf, L"Video Capture");
  // 追加を行ったのでキャプチャフィルタの参照をリリース
    pbf -> Release();
 
  // ---- グラバフィルタの準備 ----
  // グラバフィルタを作る
  CoCreateInstance( CLSID_SampleGrabber, NULL, CLSCTX_INPROC_SERVER, 
                      IID_IBaseFilter, (LPVOID *)&pF);
  pF -> QueryInterface( IID_ISampleGrabber, (void **)&pGrab );
 
  // グラバフィルタの挿入場所の特定のための設定
  AM_MEDIA_TYPE   amt;
  ZeroMemory(&amt, sizeof(AM_MEDIA_TYPE));
  amt.majortype  = MEDIATYPE_Video;
  amt.subtype    = MEDIASUBTYPE_RGB24;
  amt.formattype = FORMAT_VideoInfo; 
  pGrab -> SetMediaType( &amt );
  // グラバフィルタをフィルタグラフに追加
  pGraph -> AddFilter(pF, L"SamGra");
 
  // ---- キャプチャグラフの準備 ----
  // キャプチャグラフを作る   
  CoCreateInstance( CLSID_CaptureGraphBuilder2 , NULL, CLSCTX_INPROC,
              IID_ICaptureGraphBuilder2, (void **) &pCapture );
    // フィルタグラフをキャプチャグラフに組み込む
    pCapture -> SetFiltergraph( pGraph );
    // キャプチャグラフの設定、グラバをレンダリング出力に設定
    pCapture -> RenderStream (&PIN_CATEGORY_PREVIEW, &MEDIATYPE_Video,
                                pbf, NULL, pF);
  
  // ビットマップ情報の取得	
  pGrab -> GetConnectedMediaType( &amt ); 
  // ビデオ ヘッダーへのポインタを獲得する。
  printf( "amt.lSampleSize = %d \n", amt.lSampleSize );
  VIDEOINFOHEADER *pVideoHeader = (VIDEOINFOHEADER*)amt.pbFormat;
  // ビデオ ヘッダーには、ビットマップ情報が含まれる。
  // ビットマップ情報を BITMAPINFO 構造体にコピーする。
  BITMAPINFO BitmapInfo;
  ZeroMemory( &BitmapInfo, sizeof(BitmapInfo) );
  CopyMemory( &BitmapInfo.bmiHeader, &(pVideoHeader->bmiHeader), 
        sizeof(BITMAPINFOHEADER));
  
  bih = BitmapInfo.bmiHeader;
 
  biSizeImage = bih.biSizeImage;
 
  lpBmpData = (char *)malloc( biSizeImage );
 
 
  // opencv
 
  perFrame_img = cvCreateImage( cvSize(bih.biWidth,bih.biHeight), 8, 3 );
  temp_perFrame_img = cvCreateImage( cvSize( TSZW , TSZH), 8, 3 );
  bg_img = cvCreateImage( cvSize( TSZW , TSZH ), 8, 3 );
 
  cvZero( perFrame_img );
  cvZero( temp_perFrame_img );
  cvZero( bg_img );
}
 
void DirectShow::start(){
  pGrab -> SetBufferSamples(TRUE);  // グラブ開始
   pMC -> Run();
}
 
void DirectShow::Release(){
  pMC -> Stop();
  pGrab -> SetBufferSamples( 0 );
  // ---- 終了処理 ---- 
    // インターフェースのリリースj
    pMC -> Release();    
  pDevEnum -> Release();
    pGraph -> Release();
    pCapture -> Release();
  CoUninitialize();
}
 
////////////////////////////////////////////////////
// convert to bitmap
////////////////////////////////////////////////////
 
#ifdef REVERSE
 
inline static void ConvertToBitmap(
  const unsigned char* src,
  unsigned char* dest,
  int step,
  int width,
  int height)
{
  unsigned char* i = (unsigned char*)src;
  unsigned char* d;
  int  rowSize = width*3;
  int x, y;
 
    for( y = 0; y < height; y++ ){
    d = dest+(height-y-1)*rowSize+(rowSize-3);
        for( x = 0; x < width; x++ ){
            // fix channels, swap bgr>rgb
      d[0] = 255-i[2];
      d[1] = 255-i[1];
            d[2] = 255-i[0];
      /*d[0] = i[2];
			d[1] = i[1];
            d[2] = i[0];*/
      d-=3;
            i+=3;  // rgb
        }
    }
}
 
#else
 
inline static void ConvertToBitmap(
  const unsigned char* src,
  unsigned char* dest,
  int step,
  int width,
  int height)
{
  unsigned char* i = (unsigned char*)src;
  unsigned char* d;
  int  rowSize = width*3;
  int x, y;
 
    for( y = 0; y < height; y++ ){
    d = dest+(height-y-1)*rowSize;
        for( x = 0; x < width; x++ ){
            // fix channels, swap bgr>rgb
      d++[0] = 255-i[2];
      d++[0] = 255-i[1];
            d++[0] = 255-i[0];
            i+=3;  // rgb
        }
    }
}
 
#endif
 
////////////////////////////////////////////////////
// update shade
////////////////////////////////////////////////////
 
static IplImage* init_img = cvCreateImage( cvSize( TSZW , TSZH), 8, 3 );
static IplImage* per_img = cvCreateImage( cvSize( TSZW , TSZH), 8, 3 );
static IplImage* tmpGrayScale_img = cvCreateImage( cvSize( TSZW , TSZH), 8, 1 );
 
// parameters:
//  img - input video frame
//  dst - resultant motion picture
void DirectShow::update_shade( IplImage* frame_img, IplImage* dst_img ){
  static bool isInitCap = false;
  static int cap_counter = 0;
  if( cap_counter++ > 60 ){
    if( !isInitCap ){
      isInitCap = true;
      cvZero( init_img );
      cvCopy( frame_img, init_img );
    }
  }
 
  cvZero( per_img );
  cvCopy( frame_img, per_img );
 
  // get difference between frames
  cvAbsDiff(  init_img,  per_img,  per_img );
 
  cvSmooth( per_img, per_img, CV_GAUSSIAN, 3, 0, 0 );
  cvThreshold( per_img, per_img, 20, 255, CV_THRESH_BINARY );
 
  cvZero( tmpGrayScale_img );
  cvCvtColor( per_img, tmpGrayScale_img, CV_BGR2GRAY );
  cvCvtColor( tmpGrayScale_img, per_img, CV_GRAY2BGR );
 
  cvSmooth( per_img, per_img, CV_GAUSSIAN, 3, 0, 0 );
  cvThreshold( per_img, per_img, 1, 255, CV_THRESH_BINARY );
 
  cvCopy( per_img, dst_img );
}
 
////////////////////////////////////////////////////
// contour extraction
////////////////////////////////////////////////////
 
static IplImage* tmpShade_img = cvCreateImage( cvSize( TSZW , TSZH), IPL_DEPTH_8U, 1 );
static CvMemStorage* mst = cvCreateMemStorage(0);
 
static vector< vector<nlVector> > shadeLine;
 
void DirectShow::contourExtraction( IplImage* img ){
 
  // convert to gray scale
  cvZero( tmpShade_img );
  cvCvtColor( img, tmpShade_img, CV_BGR2GRAY);
 
  // contour extraction
  cvClearMemStorage(mst);
  CvSeq* contours = NULL;
  cvFindContours(
    tmpShade_img,
    mst,
    &contours,
    sizeof(CvContour),
    CV_RETR_LIST,
    CV_CHAIN_APPROX_NONE,
    cvPoint(0,0)
  );
 
  if( contours != NULL && contours->total != 0 ){
 
    // approach polygon
    CvSeq* Approx_contours = cvApproxPoly(
      contours,
      sizeof(CvContour),
      mst,
      CV_POLY_APPROX_DP,
      1.0,
      1
    );
 
    // initialize vector
    int i;
    for( i = 0; i < shadeLine.size(); i++ ){
      shadeLine[i].clear();
    }
    shadeLine.clear();
 
    // setup vector
    CvPoint* p;
    float px, py;
    for( ; Approx_contours != 0; Approx_contours = Approx_contours->h_next ){
      vector<nlVector> vec;
      for( i = 0; i < Approx_contours->total; i++ ){
 
        p = ((CvPoint*)cvGetSeqElem( Approx_contours, i ));
        px = 1024-p->x*1024/256;
        py = p->y*768/256;
 
        vec.push_back( nlVector( px, py ) );
      }
      shadeLine.push_back( vec );
    }
  }
}
 
void DirectShow::shade(){
  int i, j;
  nlVector* v;
  for( i = 0; i < shadeLine.size(); i++ ){
    glBegin( GL_LINE_LOOP );
    for( j = 0; j < shadeLine[i].size(); j++ ){
      v = &shadeLine[i][j];
      glVertex2f( v->x, v->y );
    }
    glEnd();
  }
}
 
////////////////////////////////////////////////////
// calculation motion flow
////////////////////////////////////////////////////
 
#define MHI_DURATION 1
#define MAX_TIME_DELTA 0.1
#define MIN_TIME_DELTA 0.05
#define OCV_N 2
#define SEQ_NUM_MAX 5
 
// ring image buffer
static IplImage **buff = NULL;
static int lastIdx = 0;
 
// temporary images
static IplImage* mhi = cvCreateImage( cvSize( TSZW , TSZH ), IPL_DEPTH_32F, 1 );
static IplImage* mask = cvCreateImage( cvSize( TSZW , TSZH ), IPL_DEPTH_8U, 1 );
static IplImage* segmask = cvCreateImage( cvSize( TSZW , TSZH ), IPL_DEPTH_32F, 1 );
static IplImage* orient = cvCreateImage( cvSize( TSZW , TSZH ), IPL_DEPTH_32F, 1 );
 
// parameters:
//  img - input video frame
void DirectShow::calcMotionFlow( IplImage* img ){
 
  if( buff == NULL ){
 
    buff = (IplImage**)malloc(OCV_N*sizeof(buff[0]) );
    memset( buff, 0, OCV_N*sizeof(buff[0]) );
 
    int i;
    for( i = 0; i < OCV_N; i++ ) {
      buff[i] = cvCreateImage( cvSize( TSZW , TSZH ), IPL_DEPTH_8U, 1 );
      cvZero( buff[i] );
 
      cvZero( mhi );
      cvZero( orient );
      cvZero( segmask );
      cvZero( mask );
    }
  }
 
  // convert frame to grayscale
  cvCvtColor( img, buff[lastIdx], CV_BGR2GRAY );
 
  int idx1, idx2;
  idx1 = lastIdx;
  idx2 = (lastIdx+1) % OCV_N;
  lastIdx = idx2;
 
  // get difference between frames
  IplImage* silh = buff[idx2];
  cvAbsDiff( buff[idx1], buff[idx2], silh );
    
  // and threshold it
  cvThreshold( silh, silh, 30, 1, CV_THRESH_BINARY );
  // update MHI
  double timestamp = clock()/1000.;
  cvUpdateMotionHistory( silh, mhi, timestamp, MHI_DURATION );
 
  // calculate motion gradient orientation and valid orientation mask
    cvCalcMotionGradient( mhi, mask, orient, MAX_TIME_DELTA, MIN_TIME_DELTA, 3 );
 
  // segment motion: get sequence of motion components
    // segmask is marked motion components map. It is not used further
  cvClearMemStorage(mst);
    CvSeq* seq = cvSegmentMotion( mhi, segmask, mst, timestamp, MAX_TIME_DELTA );
 
  // clear motion flow
  mfv.clear();
 
  int i, seq_count;
  CvRect comp_rect;
  float delta, angle, count, x, y, vx, vy;
  for( i = 0, seq_count = 0; i < seq->total; i++ ){
    if( seq_count >= SEQ_NUM_MAX ){
      break;
    }
 
    comp_rect = ((CvConnectedComp*)cvGetSeqElem( seq, i ))->rect;
    // reject very small components
    if( comp_rect.width + comp_rect.height < 100 ){
      continue;
    }
 
    angle = cvCalcGlobalOrientation( orient, mask, mhi, timestamp, MHI_DURATION );
        angle = 360.0 - angle;  // adjust for images with top-left origin
 
    count = cvNorm( silh, 0, CV_L1, 0 ); // calculate number of points within silhouette
 
    cvResetImageROI( mhi );
        cvResetImageROI( orient );
        cvResetImageROI( mask );
        cvResetImageROI( silh );
 
    // check for the case of little motion
    if( count < comp_rect.width*comp_rect.height * 0.05 ){
            continue;
    }
 
    delta = sqrt( (double)comp_rect.width*comp_rect.width+comp_rect.height*comp_rect.height );
    x = (float)comp_rect.x+comp_rect.width/2;
    y = (float)comp_rect.y+comp_rect.height/2;
    vx = delta*cos(angle*CV_PI/180);
    vy = delta*sin(angle*CV_PI/180);
    mfv.push_back( nlRectangle( x, y, vx, vy ) );
    seq_count++;
  }
}
 
////////////////////////////////////////////////////
// calculation optical flow
////////////////////////////////////////////////////
 
/*
static IplImage* current_img = cvCreateImage( cvSize( TSZW ,TSZH ), 8, 1 );
static IplImage* last_img = cvCreateImage( cvSize( TSZW ,TSZH ), 8, 1 );
static IplImage* vel_x = cvCreateImage( cvSize( TSZW ,TSZH ), IPL_DEPTH_32F, 1 );
static IplImage* vel_y = cvCreateImage( cvSize( TSZW ,TSZH ), IPL_DEPTH_32F, 1 );
 
void DirectShow::calcOpticalFlow( IplImage* img ){
 
	// convert gray scale
	cvCvtColor( img, current_img, CV_BGR2GRAY );
 
	// optical flow
	cvCalcOpticalFlowLK( last_img, current_img, cvSize( 11, 11), vel_x, vel_y );
 
	int x, y, dx, dy;
	for ( y = 0; y < TSZH; y += 10 ){
		for ( x = 0; x < TSZW; x += 10 ){
			dx = (int)cvGetReal2D( vel_x, y, x );
			dy = (int)cvGetReal2D( vel_y, y, x );
 
			cvLine( bg_img, cvPoint( x, y ), cvPoint( x+dx, y+dy), CV_RGB( 255, 255, 255), 1, 8, 0 );
			cvRectangle( bg_img, cvPoint( x+dx-1, y+dy-1), cvPoint( x+dx+1, y+dy+1),
			CV_RGB( 255, 255, 255), 1, 8, 0);
		}
	}
 
	// copy
	cvCopy( current_img, last_img );
}
 
*/
 
////////////////////////////////////////////////////
// process per frame
////////////////////////////////////////////////////
 
void DirectShow::proc(){
  HRESULT hr = pGrab -> GetCurrentBuffer( &biSizeImage, (long *)lpBmpData );
 
  if( biSizeImage > 0 ){
 
    perFrame_img->imageData = lpBmpData;
 
    // resize to texture size
    cvResize( perFrame_img, temp_perFrame_img, CV_INTER_NN );
 
    // update shade
    update_shade( temp_perFrame_img, bg_img );
 
    // contour extraction
    contourExtraction( bg_img );
 
    // calculation motion flow
    static mf_timer = 0;
    if( mf_timer++ > 2 ){
      mf_timer = 0;
      calcMotionFlow( temp_perFrame_img );
    }
 
    // calculation optical flow
    //calcOpticalFlow( temp_perFrame_img );
 
    // create texture bitmap image
    if( bg_img != NULL ){
      ConvertToBitmap(
        (unsigned char*)bg_img->imageData,
        this->btmp,
        bg_img->widthStep,
        this->_cWidth,
        this->_cHeight
      );
    }
  }
}