Gesture Recognition with gloves

4 views (last 30 days)
Priyana Kumar
Priyana Kumar on 21 Apr 2021
Hello!! I have this code that currently detects hand gestures 1-6 and completes a mouse action with it. We now have to get it to work with any color latex gloves and this is seeming to be a problem. I'm guessing it has to do with "ReturnedColorSpace" but I'm not completely sure. Currently it works well with any skin tone against a dark background with good lighting.
%% Import Java Robot
import java.awt.Robot;
import java.awt.event.*
mouse = Robot;
numFrame = 3000;
%% Create video input object.
cam = imaqhwinfo; % Get Camera information
cameraName = char(cam.InstalledAdaptors(end));
cameraInfo = imaqhwinfo(cameraName);
%cameraId = cameraInfo.DeviceInfo.DeviceID(end);
cameraFormat = char(cameraInfo.DeviceInfo(1).SupportedFormats(end)); %change 1 to 2
vidDevice = imaq.VideoDevice(cameraName, 1, cameraFormat, ... % Input Video from current adapter %change 1 to 2
'ReturnedColorSpace', 'RGB');
%vidDevice = imaq.VideoDevice('macvideo',3);
vidInfo = imaqhwinfo(vidDevice); % Acquire video information
screenSize = get(0,'ScreenSize'); % Acquire system screensize
hblob = vision.BlobAnalysis('AreaOutputPort', false, ... % Setup blob analysis handling
'CentroidOutputPort', true, ...
'BoundingBoxOutputPort', true, ...
'MinimumBlobArea', 3000, ...
'MaximumCount', 6);
palmblob = vision.BlobAnalysis('AreaOutputPort', false, ...
'CentroidOutputPort', true, ...
'BoundingBoxOutputPort', true, ...
'MinimumBlobArea', 3000, ...
'MaximumCount', 1);
hshape = vision.ShapeInserter('BorderColor','Custom', ... % Setup colored box handling
'CustomBorderColor', [1 0 0],...
'Fill', true, ...
'FillColor','Custom', 'CustomFillColor',[1 0 0], ...
'Opacity', 0.4);
palmshape = vision.ShapeInserter('BorderColor','Custom', ... % Setup colored box handling
'CustomBorderColor', [1 1 0],...
'Fill', true, ...
'FillColor','Custom', 'CustomFillColor',[1 1 0], ...
'Opacity', 0.4);
hVideoIn = vision.VideoPlayer('Name', 'Gesture Recognition', ... % Setup output video stream handling
'Position', [100 100 vidInfo.MaxWidth+20 vidInfo.MaxHeight+30]); %%%%%%%%%%%%%%%%%%%
%initializing variables
nFrame = 0;
lCount = 0; rCount = 0; dCount = 0;
sureEvent = 5;
iPos = vidInfo.MaxWidth/2;
while (nFrame < numFrame)
img1 = step(vidDevice);
%img1 = flip(img1,2);
img2 = rgb2gray(img1);
lvl = graythresh(img2);
img3 = imbinarize(img2, lvl);
img4 = bwareaopen(img3, 500);
img5 = medfilt2(img4, [5 5]);
img6 = imfill(img5,'holes');
img7 = imtophat(img6,strel('disk',35));
img8 = imopen(img7, strel('disk',35));
[cc_1,bb_1] = step(palmblob,img8); %palm
[cc_2,bb_2] = step(hblob,img7); %fingers
[B,L] = bwboundaries(img7, 'noholes');
nof = num2str(size(bb_2,1));
nof1 = size(bb_2,1);
CHAR_STR = ' not identified';
if nof1 == 1
CHAR_STR = ' MOUSE MOVE';
elseif nof1 == 2
CHAR_STR = ' LEFT CLICK';
elseif nof1 == 3
CHAR_STR = ' RIGHT CLICK';
elseif nof1 == 4
CHAR_STR = ' DOUBLE CLICK';
elseif nof1 == 5
CHAR_STR = ' SCROLL';
elseif nof1 == 6
CHAR_STR = ' QUIT VERIFICATION';
elseif nof1 == 7
CHAR_STR = ' QUIT';
end
if(~isempty(bb_2))
img1 = step(hshape,img1,bb_2);
img1 = insertText(img1,[15,5],strcat('count: ', nof,' mouse action: ',CHAR_STR),...
'FontSize', 30, 'BoxColor', [0 0 1], 'TextColor', 'white');
end
if (~isempty(bb_1))
img1 = step(palmshape,img1,bb_1);
end
imshow(img1);
movegui('northeast')
WinOnTop();
% Begin Mouse Movements
if nof1==1 %move
mouse.mouseMove(1.5*(cc_2(:,1))*screenSize(3)/vidInfo.MaxWidth, 1.5*(cc_2(:,2))*screenSize(4)/vidInfo.MaxHeight);
elseif nof1==2 %leftclick
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
pause(0.2);
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
elseif nof1==3 %rightclick brings up menu
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
pause(0.2)
mouse.mousePress(4);
pause(0.1);
mouse.mouseRelease(4);
elseif nof1==4 %doubleclick
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
pause(0.9);
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
pause(0.09);
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
lCount = 0; rCount = 0; dCount = 0;
elseif nof1==5 %scroll
mouse.mousePress(16);
pause(0.1);
mouse.mouseRelease(16);
mouse.mouseWheel(10);
mouse.mouseWheel(-10);
elseif nof1==6 %endcode, see function at the bottom
GUI_1()
pause(5);
end
end
nFrame = nFrame + 1;
%% Clearing Memory
release(hVideoIn); % Release all memory and buffer used
release(vidDevice);
clc;
function wasOnTop = WinOnTop( figureHandle, isOnTop )
%WINONTOP allows to trigger figure's "Always On Top" state
%
%% INPUT ARGUMENTS:
%
% # figureHandle - Matlab's figure handle, scalar
% # isOnTop - logical scalar or empty array
%
%
%% USAGE:
%
% * WinOnTop( hfigure, true ); - switch on "always on top"
% * WinOnTop( hfigure, false ); - switch off "always on top"
% * WinOnTop( hfigure ); - equal to WinOnTop( hfigure,true);
% * WinOnTop(); - equal to WinOnTop( gcf, true);
% * WasOnTop = WinOnTop(...); - returns boolean value "if figure WAS on top"
% * isOnTop = WinOnTop(hfigure,[]) - get "if figure is on top" property
%
% For Matlab windows, created via `hf=uifigure()` use `uifigureOnTop()`, see:
% https://www.mathworks.com/matlabcentral/fileexchange/73134-uifigureontop
%
%% LIMITATIONS:
%
% * java enabled
% * figure must be visible
% * figure's "WindowStyle" should be "normal"
% * figureHandle should not be casted to double, if using HG2 (R2014b+)
%
% Written by Igor
% i3v@mail.ru
%% Parse Inputs
if ~exist('figureHandle','var'); figureHandle = gcf; end
assert(...
isscalar( figureHandle ) &&...
ishandle( figureHandle ) &&...
strcmp(get(figureHandle,'Type'),'figure'),...
...
'WinOnTop:Bad_figureHandle_input',...
'%s','Provided figureHandle input is not a figure handle'...
);
assert(...
strcmp('on',get(figureHandle,'Visible')),...
'WinOnTop:FigInisible',...
'%s','Figure Must be Visible'...
);
assert(...
strcmp('normal',get(figureHandle,'WindowStyle')),...
'WinOnTop:FigWrongWindowStyle',...
'%s','WindowStyle Must be Normal'...
);
if ~exist('isOnTop','var'); isOnTop=true; end
assert(...
islogical( isOnTop ) && ...
isscalar( isOnTop ) || ...
isempty( isOnTop ), ...
...
'WinOnTop:Bad_isOnTop_input',...
'%s','Provided isOnTop input is neither boolean, nor empty'...
);
%% Pre-checks
error(javachk('swing',mfilename)) % Swing components must be available.
%% Action
% Flush the Event Queue of Graphic Objects and Update the Figure Window.
drawnow expose
warnStruct=warning('off','MATLAB:HandleGraphics:ObsoletedProperty:JavaFrame');
jFrame = get(handle(figureHandle),'JavaFrame');
warning(warnStruct.state,'MATLAB:HandleGraphics:ObsoletedProperty:JavaFrame');
drawnow
if ishg2(figureHandle)
jFrame_fHGxClient = jFrame.fHG2Client;
else
jFrame_fHGxClient = jFrame.fHG1Client;
end
wasOnTop = jFrame_fHGxClient.getWindow.isAlwaysOnTop;
if ~isempty(isOnTop)
jFrame_fHGxClient.getWindow.setAlwaysOnTop(isOnTop);
end
end
function tf = ishg2(figureHandle)
tf = isa(figureHandle,'matlab.ui.Figure');
end
function [] = GUI_1()
S.fh = figure('units','pixels',...
'position',[1000 25 400 200],...
'menubar','none',...
'name','Warning',...
'numbertitle','off',...
'resize','off');
S.pb = uicontrol('style','text',...
'unit','pix',...
'position',[100 60 200 100],...
'fontsize',14,...
'string',{'Are you sure you want to quit?'});
S.pb1 = uicontrol('style','push',...
'unit','pix',...
'position',[10 10 180 40],...
'fontsize',14,...
'string','Yes Quit',...
'callback',{@pb1_call,S});
S.pb2 = uicontrol('style','push',...
'units','pix',...
'position',[200 10 180 40],...
'fontsize',14,...
'string','No, Continue',...
'callback',{@pb2_call,S});
function [] = pb1_call(varargin)
quit
end
function [] = pb2_call(varargin)
close(S.fh)
end
end
Thank you in advance!

Answers (0)

Community Treasure Hunt

Find the treasures in MATLAB Central and discover how the community can help you!

Start Hunting!