微信小程序引入TensorflowJS实现Pose-detection

wechat tenserflow js movenet

添加TensorflowJS插件

登录小程序管理后台，打开“第三方设置”，添加插件TensorflowJS。本插件封装了TensorFlow.js库，利用小程序WebGL API给第三方小程序调用时提供GPU加速。

在app.json里添加插件配置：

"plugins": {
  "tfjsPlugin": {
    "version": "0.2.0",
    "provider": "wx6afed118d9e81df9"
  }
}

引入TensorFlow.js npm包

在小程序根目录添加package.json：

{
  "name": "yourProject",
  "version": "0.0.1",
  "main": "dist/index.js",
  "license": "Apache-2.0",
  "dependencies": {
    "@tensorflow-models/pose-detection": "^2.0.0",
    "@tensorflow/tfjs-backend-webgl": "^3.21.0",
    "@tensorflow/tfjs-converter": "^3.21.0",
    "@tensorflow/tfjs-core": "^3.21.0",
    "fetch-wechat": "^0.0.3"
  }
}

先npm install安装，再小程序开发者工具构建npm，关闭“将JS编译成ES5”的选项。

如果运行时出现找不到pose包的错误，尝试npm install @mediapipe/pose
（会自动引入最新版本"@mediapipe/pose": "^0.5.1635988162"）

在app.onLaunch调用configPlugin插件初始化

app.onLaunch = function () {
  tfjsPlugin.configPlugin({
    fetchFunc: fetchWechat.fetchFunc(),// polyfill fetch function
    tf,// inject tfjs runtime
    webgl,// inject backend
    canvas: wx.createOffscreenCanvas(),// provide webgl canvas
  });
}

下载模型文件

tensorflow的预训练模型是放在墙外，需要下载下来放可访问的服务器上：
https://tfhub.dev/google/tfjs-model/movenet/singlepose/lightning/4

加载Movenet模型

const poseDetection = require('@tensorflow-models/pose-detection')
page.onLoad = async function(){
  const modelUrl = 'https://<host>/tfjs/tfjs-model/movenet/singlepose/lightning/4/model.json'
  const modelConfig = {
    modelUrl,
    modelType: poseDetection.movenet.modelType.SINGLEPOSE_LIGHTNING,
  };
  this.movenet = await poseDetection.createDetector(poseDetection.SupportedModels.MoveNet, modelConfig).catch(console.error)
}

使用摄像头逐帧识别

在页面wxml添加camera组件

<camera device-position="front" flash="off"></camera>
<button type="primary" bindtap="poseDetect">识别</button>

page.poseDetect = function(){
  this.data.ts = Date.now()
  this.cameraCtx = this.cameraCtx || wx.createCameraContext()
  //侦听相机视频流
  this.cameraLisenter = this.cameraLisenter || this.cameraCtx.onCameraFrame(frame => {
    if (this.estimatePoses.posing || Date.now() < this.data.ts+100) return//限速
    this.estimatePoses(frame).then(() => this.data.ts = Date.now())
  })
  //toggle start/stop posing
  if (!this.data.posing){
    this.cameraLisenter && this.cameraLisenter.start()
  }else{
    this.cameraLisenter && this.cameraLisenter.stop()
  }
  this.data.posing = !this.data.posing
}
page.estimatePoses = async function(frame){
  if (this.estimatePoses.posing) return//限流
  this.estimatePoses.posing = true
  const data = {//把视频帧转为estimatePoses要求的格式
      data: new Uint8Array(frame.data),
      height: Number(frame.height),
      width: Number(frame.width)
  }
  console.time('posing')
  const poses = await this.movenet.estimatePoses(data).catch(console.error)
  if (poses.length){//检测到人体
    // this.drawPoses(poses[0].keypoints)
    console.log(poses)
  }
  console.timeEnd('posing')
  this.estimatePoses.posing = false
}