Skip to content

Commit 7aca8b3

Browse files
jesse-nsquaredjesse-nsquared
committed
lab05
1 parent c3532b1 commit 7aca8b3

File tree

3 files changed

+163
-8
lines changed

3 files changed

+163
-8
lines changed

‎Kinect2Sample/Kinect2Sample.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@
2626
<UseVSHostingProcess>false</UseVSHostingProcess>
2727
<ErrorReport>prompt</ErrorReport>
2828
<Prefer32Bit>true</Prefer32Bit>
29+
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
30+
<Optimize>true</Optimize>
2931
</PropertyGroup>
3032
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
3133
<OutputPath>bin\x64\Release\</OutputPath>

‎Kinect2Sample/MainPage.xaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
<Setter Property="FontSize" Value="20"/>
2626
</Style>
2727
</Page.Resources>
28-
28+
2929
<Grid Background="{ThemeResource ApplicationPageBackgroundThemeBrush}" >
3030
<Grid Margin="30">
3131
<Grid.RowDefinitions>
@@ -54,6 +54,10 @@
5454
Click="ColorButton_Click"/>
5555
<Button Content="Depth" Style="{StaticResource FrameSelectorButtonStyle}"
5656
Click="DepthButton_Click"/>
57+
<Button Style="{StaticResource FrameSelectorButtonStyle}"
58+
Click="BodyMaskButton_Click">
59+
<TextBlock Text="Body Mask" TextWrapping="Wrap"/>
60+
</Button>
5761
</StackPanel>
5862
</ScrollViewer>
5963
</Grid>

‎Kinect2Sample/MainPage.xaml.cs‎

Lines changed: 156 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,18 @@
1515
using Windows.UI.Xaml.Navigation;
1616
using WindowsPreview.Kinect;
1717
using System.ComponentModel;
18+
using Windows.Storage.Streams;
19+
using System.Runtime.InteropServices;
20+
using System.Diagnostics;
1821

1922
namespace Kinect2Sample
2023
{
2124
public enum DisplayFrameType
2225
{
2326
Infrared,
2427
Color,
25-
Depth
28+
Depth,
29+
BodyMask
2630
}
2731

2832
public sealed partial class MainPage : Page, INotifyPropertyChanged
@@ -76,6 +80,7 @@ public sealed partial class MainPage : Page, INotifyPropertyChanged
7680
private FrameDescription currentFrameDescription;
7781
private DisplayFrameType currentDisplayFrameType;
7882
private MultiSourceFrameReader multiSourceFrameReader = null;
83+
private CoordinateMapper coordinateMapper = null;
7984

8085
//Infrared Frame
8186
private ushort[] infraredFrameData = null;
@@ -85,6 +90,9 @@ public sealed partial class MainPage : Page, INotifyPropertyChanged
8590
private ushort[] depthFrameData = null;
8691
private byte[] depthPixels = null;
8792

93+
//BodyMask Frames
94+
private DepthSpacePoint[] colorMappedToDepthPoints = null;
95+
8896
public event PropertyChangedEventHandler PropertyChanged;
8997
public string StatusText
9098
{
@@ -125,7 +133,9 @@ public MainPage()
125133

126134
SetupCurrentDisplay(DEFAULT_DISPLAYFRAMETYPE);
127135

128-
this.multiSourceFrameReader = this.kinectSensor.OpenMultiSourceFrameReader(FrameSourceTypes.Infrared | FrameSourceTypes.Color | FrameSourceTypes.Depth);
136+
this.coordinateMapper = this.kinectSensor.CoordinateMapper;
137+
138+
this.multiSourceFrameReader = this.kinectSensor.OpenMultiSourceFrameReader(FrameSourceTypes.Infrared | FrameSourceTypes.Color | FrameSourceTypes.Depth | FrameSourceTypes.BodyIndex);
129139

130140
this.multiSourceFrameReader.MultiSourceFrameArrived += this.Reader_MultiSourceFrameArrived;
131141

@@ -144,6 +154,8 @@ public MainPage()
144154
private void SetupCurrentDisplay(DisplayFrameType newDisplayFrameType)
145155
{
146156
currentDisplayFrameType = newDisplayFrameType;
157+
// Frames used by more than one type are declared outside the switch
158+
FrameDescription colorFrameDescription = null;
147159
switch (currentDisplayFrameType)
148160
{
149161
case DisplayFrameType.Infrared:
@@ -156,7 +168,7 @@ private void SetupCurrentDisplay(DisplayFrameType newDisplayFrameType)
156168
break;
157169

158170
case DisplayFrameType.Color:
159-
FrameDescription colorFrameDescription = this.kinectSensor.ColorFrameSource.FrameDescription;
171+
colorFrameDescription = this.kinectSensor.ColorFrameSource.FrameDescription;
160172
this.CurrentFrameDescription = colorFrameDescription;
161173
// create the bitmap to display
162174
this.bitmap = new WriteableBitmap(colorFrameDescription.Width, colorFrameDescription.Height);
@@ -171,6 +183,13 @@ private void SetupCurrentDisplay(DisplayFrameType newDisplayFrameType)
171183
this.bitmap = new WriteableBitmap(depthFrameDescription.Width, depthFrameDescription.Height);
172184
break;
173185

186+
case DisplayFrameType.BodyMask:
187+
colorFrameDescription = this.kinectSensor.ColorFrameSource.FrameDescription;
188+
this.CurrentFrameDescription = colorFrameDescription;
189+
// allocate space to put the pixels being received and converted
190+
this.colorMappedToDepthPoints = new DepthSpacePoint[colorFrameDescription.Width * colorFrameDescription.Height];
191+
this.bitmap = new WriteableBitmap(colorFrameDescription.Width, colorFrameDescription.Height);
192+
break;
174193
default:
175194
break;
176195
}
@@ -183,40 +202,158 @@ private void Sensor_IsAvailableChanged(KinectSensor sender, IsAvailableChangedEv
183202

184203
private void Reader_MultiSourceFrameArrived(MultiSourceFrameReader sender, MultiSourceFrameArrivedEventArgs e)
185204
{
205+
186206
MultiSourceFrame multiSourceFrame = e.FrameReference.AcquireFrame();
187207

188208
// If the Frame has expired by the time we process this event, return.
189209
if (multiSourceFrame == null)
190210
{
191211
return;
192212
}
213+
DepthFrame depthFrame = null;
214+
ColorFrame colorFrame = null;
215+
InfraredFrame infraredFrame = null;
216+
BodyIndexFrame bodyIndexFrame = null;
217+
//windows.storage.streams
218+
IBuffer depthFrameData = null;
219+
IBuffer bodyIndexFrameData = null;
220+
// Com interface for unsafe byte manipulation
221+
IBufferByteAccess bodyIndexByteAccess = null;
193222

194223
switch (currentDisplayFrameType)
195224
{
196225
case DisplayFrameType.Infrared:
197-
using (InfraredFrame infraredFrame = multiSourceFrame.InfraredFrameReference.AcquireFrame())
226+
using (infraredFrame = multiSourceFrame.InfraredFrameReference.AcquireFrame())
198227
{
199228
ShowInfraredFrame(infraredFrame);
200229
}
201230
break;
202231
case DisplayFrameType.Color:
203-
using (ColorFrame colorFrame = multiSourceFrame.ColorFrameReference.AcquireFrame())
232+
using (colorFrame = multiSourceFrame.ColorFrameReference.AcquireFrame())
204233
{
205234
ShowColorFrame(colorFrame);
206235
}
207236
break;
208237
case DisplayFrameType.Depth:
209-
using (DepthFrame depthFrame =
210-
multiSourceFrame.DepthFrameReference.AcquireFrame())
238+
using (depthFrame = multiSourceFrame.DepthFrameReference.AcquireFrame())
211239
{
212240
ShowDepthFrame(depthFrame);
213241
}
214242
break;
243+
case DisplayFrameType.BodyMask:
244+
// Put in a try catch to utilise finally() and clean up frames
245+
try
246+
{
247+
depthFrame = multiSourceFrame.DepthFrameReference.AcquireFrame();
248+
bodyIndexFrame = multiSourceFrame.BodyIndexFrameReference.AcquireFrame();
249+
colorFrame = multiSourceFrame.ColorFrameReference.AcquireFrame();
250+
if ((depthFrame == null) || (colorFrame == null) || (bodyIndexFrame == null))
251+
{
252+
return;
253+
}
254+
255+
// Access the depth frame data directly via LockImageBuffer to avoid making a copy
256+
depthFrameData = depthFrame.LockImageBuffer();
257+
this.coordinateMapper.MapColorFrameToDepthSpaceUsingIBuffer(depthFrameData, this.colorMappedToDepthPoints);
258+
// Process Color
259+
colorFrame.CopyConvertedFrameDataToBuffer(this.bitmap.PixelBuffer, ColorImageFormat.Bgra);
260+
// Access the body index frame data directly via LockImageBuffer to avoid making a copy
261+
bodyIndexFrameData = bodyIndexFrame.LockImageBuffer();
262+
ShowMappedBodyFrame(depthFrame.FrameDescription.Width, depthFrame.FrameDescription.Height, bodyIndexFrameData, bodyIndexByteAccess);
263+
264+
}
265+
finally
266+
{
267+
// ... disposing of depth, color and bodyIndex frames
268+
if (depthFrame != null)
269+
{
270+
depthFrame.Dispose();
271+
}
272+
if (colorFrame != null)
273+
{
274+
colorFrame.Dispose();
275+
}
276+
if (bodyIndexFrame != null)
277+
{
278+
bodyIndexFrame.Dispose();
279+
}
280+
281+
if (depthFrameData != null)
282+
{
283+
// We must force a release of the IBuffer in order to ensure that we have dropped all references to it.
284+
System.Runtime.InteropServices.Marshal.ReleaseComObject(depthFrameData);
285+
}
286+
if (bodyIndexFrameData != null)
287+
{
288+
System.Runtime.InteropServices.Marshal.ReleaseComObject(bodyIndexFrameData);
289+
}
290+
if (bodyIndexByteAccess != null)
291+
{
292+
System.Runtime.InteropServices.Marshal.ReleaseComObject(bodyIndexByteAccess);
293+
}
294+
295+
}
296+
break;
297+
215298
default:
216299
break;
217300
}
218301
}
219302

303+
unsafe private void ShowMappedBodyFrame(int depthWidth, int depthHeight, IBuffer bodyIndexFrameData, IBufferByteAccess bodyIndexByteAccess)
304+
{
305+
bodyIndexByteAccess = (IBufferByteAccess)bodyIndexFrameData;
306+
byte* bodyIndexBytes = null;
307+
bodyIndexByteAccess.Buffer(out bodyIndexBytes);
308+
309+
fixed (DepthSpacePoint* colorMappedToDepthPointsPointer = this.colorMappedToDepthPoints)
310+
{
311+
IBufferByteAccess bitmapBackBufferByteAccess = (IBufferByteAccess)this.bitmap.PixelBuffer;
312+
313+
byte* bitmapBackBufferBytes = null;
314+
bitmapBackBufferByteAccess.Buffer(out bitmapBackBufferBytes);
315+
316+
// Treat the color data as 4-byte pixels
317+
uint* bitmapPixelsPointer = (uint*)bitmapBackBufferBytes;
318+
319+
// Loop over each row and column of the color image
320+
// Zero out any pixels that don't correspond to a body index
321+
int colorMappedLength = this.colorMappedToDepthPoints.Length;
322+
for (int colorIndex = 0; colorIndex < colorMappedLength; ++colorIndex)
323+
{
324+
float colorMappedToDepthX = colorMappedToDepthPointsPointer[colorIndex].X;
325+
float colorMappedToDepthY = colorMappedToDepthPointsPointer[colorIndex].Y;
326+
327+
// The sentinel value is -inf, -inf, meaning that no depth pixel corresponds to this color pixel.
328+
if (!float.IsNegativeInfinity(colorMappedToDepthX) &&
329+
!float.IsNegativeInfinity(colorMappedToDepthY))
330+
{
331+
// Make sure the depth pixel maps to a valid point in color space
332+
int depthX = (int)(colorMappedToDepthX + 0.5f);
333+
int depthY = (int)(colorMappedToDepthY + 0.5f);
334+
335+
// If the point is not valid, there is no body index there.
336+
if ((depthX >= 0) && (depthX < depthWidth) && (depthY >= 0) && (depthY < depthHeight))
337+
{
338+
int depthIndex = (depthY * depthWidth) + depthX;
339+
340+
// If we are tracking a body for the current pixel, do not zero out the pixel
341+
if (bodyIndexBytes[depthIndex] != 0xff)
342+
{
343+
// this bodyIndexByte is good and is a body, loop again.
344+
continue;
345+
}
346+
}
347+
}
348+
// this pixel does not correspond to a body so make it black and transparent
349+
bitmapPixelsPointer[colorIndex] = 0;
350+
}
351+
}
352+
353+
this.bitmap.Invalidate();
354+
FrameDisplayImage.Source = this.bitmap;
355+
}
356+
220357
private void ShowDepthFrame(DepthFrame depthFrame)
221358
{
222359
bool depthFrameProcessed = false;
@@ -387,5 +524,17 @@ private void DepthButton_Click(object sender, RoutedEventArgs e)
387524
SetupCurrentDisplay(DisplayFrameType.Depth);
388525
}
389526

527+
private void BodyMaskButton_Click(object sender, RoutedEventArgs e)
528+
{
529+
SetupCurrentDisplay(DisplayFrameType.BodyMask);
530+
}
531+
532+
[Guid("905a0fef-bc53-11df-8c49-001e4fc686da"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
533+
interface IBufferByteAccess
534+
{
535+
unsafe void Buffer(out byte* pByte);
536+
}
537+
538+
390539
}
391540
}

0 commit comments

Comments
 (0)