Skip to content

Commit 8717622

Browse files
ardoerlemansardoerlemans
authored
Adding MoveNet to Pose Detection API (#627)
* Adding MoveNet skeleton and support files * Adding MoveNet implementation and supporting files * Lint MoveNet files (except for line length) Line lengths will be updated in an upcoming commit. * Use correct formatting, which includes line lengths * Remove test URL * Addressing code review comments * Fix MoveNet visualization in live demo * Updates after review comments MoveNet determineCropRegion refactored and moved OneEuroFilter to common filters directory * Remove configurable keypoint threshold This threshold was mostly used for internal cropping logic. * Merge Model and KeypointModel, plus additional updates after review * Move one euro filter back to MoveNet with TODO * Add test for MoveNet * Use MoveNet as default model in pose demo * Add MoveNet model type selection to pose demo * Run models at highest possible speed in pose demo * Fix error in cropping code * Remove comment * Updates to resolve review comments * Fix CI test * Simplify pose array creation
1 parent 2b3484c commit 8717622

16 files changed

+826
-31
lines changed

‎pose-detection/demo/src/camera.js‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,6 @@ export class Camera {
2424
this.video = document.getElementById('video');
2525
this.canvas = document.getElementById('output');
2626
this.ctx = this.canvas.getContext('2d');
27-
28-
// The video frame rate may be lower than the browser animate frame
29-
// rate. We use this to avoid processing the same frame twice.
30-
this.lastVideoTime = 0;
3127
}
3228

3329
/**

‎pose-detection/demo/src/index.js‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,13 @@ async function createDetector(model) {
4040
case posedetection.SupportedModels.MediapipeBlazepose:
4141
return posedetection.createDetector(
4242
STATE.model.model, {quantBytes: 4, upperBodyOnly: false});
43+
case posedetection.SupportedModels.MoveNet:
44+
const modelType =
45+
STATE.model[STATE.model.model].modelType == 'Lightning' ?
46+
posedetection.movenet.modelType.SINGLEPOSE_LIGHTNING :
47+
posedetection.movenet.modelType.SINGLEPOSE_THUNDER;
48+
return posedetection.createDetector(
49+
STATE.model.model, {modelType: modelType});
4350
}
4451
}
4552

@@ -58,28 +65,24 @@ async function checkGuiUpdate() {
5865
}
5966

6067
async function renderResult() {
61-
if (camera.video.currentTime !== camera.lastVideoTime) {
62-
camera.lastVideoTime = camera.video.currentTime;
63-
64-
// FPS only counts the time it takes to finish estimatePoses.
65-
stats.begin();
68+
// FPS only counts the time it takes to finish estimatePoses.
69+
stats.begin();
6670

67-
const poses = await detector.estimatePoses(
68-
camera.video, {maxPoses: 1, flipHorizontal: false});
71+
const poses = await detector.estimatePoses(
72+
camera.video, {maxPoses: 1, flipHorizontal: false});
6973

70-
stats.end();
74+
stats.end();
7175

72-
camera.drawCtx();
76+
camera.drawCtx();
7377

74-
// The null check makes sure the UI is not in the middle of changing to a
75-
// different model. If changeToModel is non-null, the result is from an
76-
// old model, which shouldn't be rendered.
77-
if (poses.length > 0 && STATE.changeToModel == null) {
78-
const shouldScale = STATE.model.model ===
79-
posedetection.SupportedModels.MediapipeBlazepose;
78+
// The null check makes sure the UI is not in the middle of changing to a
79+
// different model. If changeToModel is non-null, the result is from an
80+
// old model, which shouldn't be rendered.
81+
if (poses.length > 0 && STATE.changeToModel == null) {
82+
const shouldScale =
83+
STATE.model.model === posedetection.SupportedModels.MediapipeBlazepose;
8084

81-
camera.drawResult(poses[0], shouldScale);
82-
}
85+
camera.drawResult(poses[0], shouldScale);
8386
}
8487
}
8588

‎pose-detection/demo/src/option_panel.js‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,24 +44,36 @@ export function setupDatGui() {
4444
case posedetection.SupportedModels.PoseNet:
4545
poseNetFolder.open();
4646
blazePoseFolder.close();
47+
moveNetFolder.close();
4748
break;
4849
case posedetection.SupportedModels.MediapipeBlazepose:
4950
blazePoseFolder.open();
5051
poseNetFolder.close();
52+
moveNetFolder.close();
53+
break;
54+
case posedetection.SupportedModels.MoveNet:
55+
blazePoseFolder.close();
56+
poseNetFolder.close();
57+
moveNetFolder.open();
5158
break;
5259
default:
5360
throw new Error(`${model} is not supported.`);
5461
}
5562
});
5663
modelFolder.open();
5764

58-
// The PoseNet model config folder contains options for PoseNet config
65+
// The MoveNet model config folder contains options for MoveNet config
5966
// settings.
60-
const poseNetFolder = gui.addFolder('PoseNet Config');
61-
poseNetFolder.add(
62-
STATE.model[posedetection.SupportedModels.PoseNet], 'scoreThreshold', 0,
67+
const moveNetFolder = gui.addFolder('MoveNet Config');
68+
const moveNetTypeController = moveNetFolder.add(
69+
STATE.model[posedetection.SupportedModels.MoveNet], 'modelType',
70+
['Thunder', 'Lightning']);
71+
moveNetTypeController.onChange(type => {
72+
STATE.changeToModel = type;
73+
});
74+
moveNetFolder.add(
75+
STATE.model[posedetection.SupportedModels.MoveNet], 'scoreThreshold', 0,
6376
1);
64-
poseNetFolder.open();
6577

6678
// The Blazepose model config folder contains options for Blazepose config
6779
// settings.
@@ -70,5 +82,14 @@ export function setupDatGui() {
7082
STATE.model[posedetection.SupportedModels.MediapipeBlazepose],
7183
'scoreThreshold', 0, 1);
7284

85+
// The PoseNet model config folder contains options for PoseNet config
86+
// settings.
87+
const poseNetFolder = gui.addFolder('PoseNet Config');
88+
poseNetFolder.add(
89+
STATE.model[posedetection.SupportedModels.PoseNet], 'scoreThreshold', 0,
90+
1);
91+
92+
moveNetFolder.open();
93+
7394
return gui;
7495
}

‎pose-detection/demo/src/params.js‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ export const VIDEO_SIZE = {
2626
export const STATE = {
2727
camera: {targetFPS: 60, sizeOption: '640 X 480'},
2828
model: {
29-
model: posedetection.SupportedModels.PoseNet,
29+
model: posedetection.SupportedModels.MoveNet,
3030
}
3131
};
3232
STATE.model[posedetection.SupportedModels.MediapipeBlazepose] = {
@@ -35,3 +35,7 @@ STATE.model[posedetection.SupportedModels.MediapipeBlazepose] = {
3535
STATE.model[posedetection.SupportedModels.PoseNet] = {
3636
scoreThreshold: 0.5
3737
};
38+
STATE.model[posedetection.SupportedModels.MoveNet] = {
39+
modelType: 'Thunder',
40+
scoreThreshold: 0.3
41+
};

‎pose-detection/src/constants.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,3 +53,22 @@ export const BLAZEPOSE_CONNECTED_KEYPOINTS_PAIRS = [
5353
[18, 20], [23, 25], [23, 24], [24, 26], [25, 27], [26, 28], [27, 29],
5454
[28, 30], [27, 31], [28, 32], [29, 31], [30, 32]
5555
];
56+
export const COCO_KEYPOINTS_NAMED_MAP: {[index: string]: number} = {
57+
nose: 0,
58+
left_eye: 1,
59+
right_eye: 2,
60+
left_ear: 3,
61+
right_ear: 4,
62+
left_shoulder: 5,
63+
right_shoulder: 6,
64+
left_elbow: 7,
65+
right_elbow: 8,
66+
left_wrist: 9,
67+
right_wrist: 10,
68+
left_hip: 11,
69+
right_hip: 12,
70+
left_knee: 13,
71+
right_knee: 14,
72+
left_ankle: 15,
73+
right_ankle: 16
74+
};

‎pose-detection/src/create_detector.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
import {BlazeposeDetector} from './blazepose/detector';
1919
import {BlazeposeModelConfig} from './blazepose/types';
20+
import {MoveNetDetector} from './movenet/detector';
21+
import {MoveNetModelConfig} from './movenet/types';
2022
import {PoseDetector} from './pose_detector';
2123
import {PosenetDetector} from './posenet/detector';
2224
import {PosenetModelConfig} from './posenet/types';
@@ -29,13 +31,15 @@ import {SupportedModels} from './types';
2931
*/
3032
export async function createDetector(
3133
model: SupportedModels,
32-
modelConfig: PosenetModelConfig|
33-
BlazeposeModelConfig): Promise<PoseDetector> {
34+
modelConfig: PosenetModelConfig|BlazeposeModelConfig|
35+
MoveNetModelConfig): Promise<PoseDetector> {
3436
switch (model) {
3537
case SupportedModels.PoseNet:
3638
return PosenetDetector.load(modelConfig as PosenetModelConfig);
3739
case SupportedModels.MediapipeBlazepose:
3840
return BlazeposeDetector.load(modelConfig as BlazeposeModelConfig);
41+
case SupportedModels.MoveNet:
42+
return MoveNetDetector.load(modelConfig as MoveNetModelConfig);
3943
default:
4044
throw new Error(`${model} is not a supported model name.`);
4145
}

‎pose-detection/src/index.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,25 @@
1818
// Entry point to create a new detector instance.
1919
export {BlazeposeEstimationConfig, BlazeposeModelConfig} from './blazepose/types';
2020
export {createDetector} from './create_detector';
21+
export {MoveNetEstimationConfig, MoveNetModelConfig} from './movenet/types';
2122
// PoseDetector class.
2223
export {PoseDetector} from './pose_detector';
2324
export {PoseNetEstimationConfig, PosenetModelConfig} from './posenet/types';
25+
2426
// Supported models enum.
2527
export * from './types';
2628

2729
// Second level exports.
2830
// Utils for rendering.
2931
import * as util from './util';
3032
export {util};
33+
34+
// MoveNet model types.
35+
import {SINGLEPOSE_LIGHTNING, SINGLEPOSE_THUNDER} from './movenet/constants';
36+
const movenet = {
37+
modelType: {
38+
'SINGLEPOSE_LIGHTNING': SINGLEPOSE_LIGHTNING,
39+
'SINGLEPOSE_THUNDER': SINGLEPOSE_THUNDER
40+
}
41+
};
42+
export {movenet};

‎pose-detection/src/movenet/constants.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/**
2+
* @license
3+
* Copyright 2021 Google LLC. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
* =============================================================================
16+
*/
17+
18+
import {MoveNetEstimationConfig, MoveNetModelConfig} from './types';
19+
20+
export const SINGLEPOSE_LIGHTNING = 'SinglePose.Lightning';
21+
export const SINGLEPOSE_THUNDER = 'SinglePose.Thunder';
22+
23+
export const VALID_MODELS = [SINGLEPOSE_LIGHTNING, SINGLEPOSE_THUNDER];
24+
25+
export const MOVENET_SINGLEPOSE_LIGHTNING_URL =
26+
'https://tfhub.dev/google/tfjs-model/movenet/singlepose/lightning/1';
27+
export const MOVENET_SINGLEPOSE_THUNDER_URL =
28+
'https://tfhub.dev/google/tfjs-model/movenet/singlepose/thunder/1';
29+
30+
export const MOVENET_SINGLEPOSE_LIGHTNING_RESOLUTION = 192;
31+
export const MOVENET_SINGLEPOSE_THUNDER_RESOLUTION = 256;
32+
33+
// The default configuration for loading MoveNet.
34+
export const MOVENET_CONFIG: MoveNetModelConfig = {
35+
modelType: SINGLEPOSE_LIGHTNING
36+
};
37+
38+
export const MOVENET_SINGLE_POSE_ESTIMATION_CONFIG: MoveNetEstimationConfig = {
39+
maxPoses: 1
40+
};
41+
42+
export const MIN_CROP_KEYPOINT_SCORE = 0.3;

0 commit comments

Comments
 (0)