lab09

Author: jesse-nsquared

Kinect2Sample/Kinect2Sample.csproj

@@ -92,6 +92,9 @@
     <None Include="Kinect2Sample_TemporaryKey.pfx" />
   </ItemGroup>
   <ItemGroup>
+    <Reference Include="DirectXSceneStore">
+      <HintPath>Libraries\x64\DirectXSceneStore.winmd</HintPath>
+    </Reference>
     <Reference Include="KinectFaceStore">
       <HintPath>Libraries\x64\KinectFaceStore.dll</HintPath>
     </Reference>

Kinect2Sample/Libraries/x64/DirectXSceneStore.dll

@@ -5,6 +5,7 @@
     xmlns:local="using:Kinect2Sample"
     xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
     xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+    xmlns:dx="using:DirectXSceneStore"
     mc:Ignorable="d">
     <Page.Resources>
         <LinearGradientBrush x:Key="ButtonGradientBrush" StartPoint="0,0" EndPoint="0,1" >
@@ -53,6 +54,10 @@
             <Viewbox Grid.Row="1" HorizontalAlignment="Center">
                 <Grid x:Name="BodyJointsGrid" Background="Transparent" Width="512" Height="414"/>
             </Viewbox>
+            <dx:ScenePanel x:Name="DXScenePanel" Grid.Row="1" Margin="20"
+                   Visibility="{Binding CurrentDisplayFrameType, 
+                   Converter={StaticResource DisplayTypeToVisibilityConverter}, 
+                   ConverterParameter=FaceGame }" />
             <Viewbox Grid.Row="1" HorizontalAlignment="Center">
                 <Canvas x:Name="FacePointsCanvas"/>
             </Viewbox>
@@ -90,9 +95,13 @@
                         <TextBlock Text="Color Face" TextWrapping="Wrap" />
                     </Button>
                     <Button Style="{StaticResource FrameSelectorButtonStyle}"
-                            Click="InfraredFaceButton_Click" x:Name="InfraredFaceButton">
+                            Click="InfraredFaceButton_Click">
                         <TextBlock Text="Infrared Face" TextWrapping="Wrap" />
                     </Button>
+                    <Button Style="{StaticResource FrameSelectorButtonStyle}"
+                            Click="FaceGameButton_Click">
+                        <TextBlock Text="Face Game" TextWrapping="Wrap" />
+                    </Button>
                 </StackPanel>
             </ScrollViewer>
         </Grid>

Kinect2Sample/Libraries/x64/DirectXSceneStore.pri

@@ -34,13 +34,17 @@ public enum DisplayFrameType
         BodyJoints,
         BackgroundRemoved,
         FaceOnColor,
-        FaceOnInfrared
+        FaceOnInfrared,
+        FaceGame
     }
 
     public sealed partial class MainPage : Page, INotifyPropertyChanged
     {
         private const DisplayFrameType DEFAULT_DISPLAYFRAMETYPE = DisplayFrameType.Infrared;
 
+        private const double FACE_AIMING_ACCURACY= 1.0;
+        private const double FACE_AIMING_SENSITIVITY = 0.01;
+
         /// <summary>
         /// The highest value that can be returned in the InfraredFrame.
         /// It is cast to a float for readability in the visualization code.
@@ -113,6 +117,10 @@ public sealed partial class MainPage : Page, INotifyPropertyChanged
         //Cat assets
         private Image[] catEyeRightOpen, catEyeRightClosed, catEyeLeftOpen, catEyeLeftClosed, catNose;
 
+        //Face Orientation Shaping
+        private double prevPitch = 0.0f;
+        private double prevYaw = 0.0f;
+
         public event PropertyChangedEventHandler PropertyChanged;
         public string StatusText
         {
@@ -391,6 +399,13 @@ private void SetupCurrentDisplay(DisplayFrameType newDisplayFrameType)
                     this.FacePointsCanvas.Height = infraredFrameDescription.Height;
                     break;
 
+                case DisplayFrameType.FaceGame:
+                    colorFrameDescription = this.kinectSensor.ColorFrameSource.FrameDescription;
+                    this.CurrentFrameDescription = colorFrameDescription;
+                    this.FacePointsCanvas.Width = colorFrameDescription.Width;
+                    this.FacePointsCanvas.Height = colorFrameDescription.Height;
+                    break;
+
                 default:
                     break;
             }
@@ -540,16 +555,69 @@ private void Reader_MultiSourceFrameArrived(MultiSourceFrameReader sender, Multi
                         DrawFaceOnInfrared();
                     }
                     break;
+                case DisplayFrameType.FaceGame:
+                    FaceGameLookUpdate();
+                    break;
                 default:
                     break;
             }
         }
 
+        private void FaceGameLookUpdate()
+        {
+            this.FacePointsCanvas.Children.Clear();
+            FaceFrameResult[] results = faceManager.GetLatestFaceFrameResults();
+            for (int i = 0; i < results.Count(); i++)
+            {
+                if (results[i] != null)
+                {
+                    foreach (KeyValuePair<FacePointType, Point> facePointKVP in
+                        results[i].FacePointsInColorSpace)
+                    {
+                        if (facePointKVP.Value.X == 0.0 || facePointKVP.Value.Y == 0.0)
+                        {
+                            break;
+                        }
+                        Size ellipseSize = new Size(10, 10);
+                        Ellipse ellipse = new Ellipse();
+                        ellipse.Width = ellipseSize.Width;
+                        ellipse.Height = ellipseSize.Height;
+                        ellipse.Fill = new SolidColorBrush(Colors.Red);
+                        Canvas.SetLeft(ellipse, facePointKVP.Value.X - (ellipseSize.Width / 2));
+                        Canvas.SetTop(ellipse, facePointKVP.Value.Y - (ellipseSize.Height / 2));
+                        this.FacePointsCanvas.Children.Add(ellipse);
+                    }
+
+                    double pitch, roll, yaw = 0;
+
+                    ExtractFaceRotationInDegrees(results[i].FaceRotationQuaternion, out pitch, out yaw, out roll);
+
+                    double pitchDiff = Math.Abs(pitch - prevPitch);
+                    double yawDiff = Math.Abs(yaw - prevYaw);
+                    if (pitchDiff > FACE_AIMING_ACCURACY ||
+                        yawDiff > FACE_AIMING_ACCURACY)
+                    {
+                        this.DXScenePanel.SetYawPitch(
+                            -(float)(yaw * FACE_AIMING_SENSITIVITY), 
+                            (float)(pitch * FACE_AIMING_SENSITIVITY));
+                        prevPitch = pitch;
+                        prevYaw = yaw;
+                    }
+
+                    if (results[i].FaceProperties[FaceProperty.MouthOpen] == DetectionResult.Yes)
+                    {
+                        this.DXScenePanel.Fire();
+                    }
+                    break;
+                }
+            }
+        }
+
         private void DrawFaceOnInfrared()
         {
             FacePointsCanvas.Children.Clear();
             FaceFrameResult[] results = faceManager.GetLatestFaceFrameResults();
-            for (int i = 0; i < results.Count(); i++ )
+            for (int i = 0; i < results.Count(); i++)
             {
                 if (results[i] != null)
                 {
@@ -869,6 +937,19 @@ private void RenderPixelArray(byte[] pixels)
             this.FrameDisplayImage.Source = this.bitmap;
         }
 
+        private static void ExtractFaceRotationInDegrees(Vector4 rotQuaternion, out double pitch, out double yaw, out double roll)
+        {
+            double x = rotQuaternion.X;
+            double y = rotQuaternion.Y;
+            double z = rotQuaternion.Z;
+            double w = rotQuaternion.W;
+
+            // convert face rotation quaternion to Euler angles in degrees
+            pitch = Math.Atan2(2 * ((y * z) + (w * x)), (w * w) - (x * x) - (y * y) + (z * z)) / Math.PI * 180.0;
+            yaw = Math.Asin(2 * ((w * y) - (x * z))) / Math.PI * 180.0;
+            roll = Math.Atan2(2 * ((x * y) + (w * z)), (w * w) + (x * x) - (y * y) - (z * z)) / Math.PI * 180.0;
+        }
+
         private void InfraredButton_Click(object sender, RoutedEventArgs e)
         {
             SetupCurrentDisplay(DisplayFrameType.Infrared);
@@ -914,5 +995,10 @@ interface IBufferByteAccess
         {
             unsafe void Buffer(out byte* pByte);
         }
+
+        private void FaceGameButton_Click(object sender, RoutedEventArgs e)
+        {
+            SetupCurrentDisplay(DisplayFrameType.FaceGame);
+        }
     }
 }