Tensorflow

src/components/Game.vue

@@ -1,10 +1,15 @@
 <template>
   <div>
     <button v-on:click="sendMessage()">Send Message</button>
+    <video ref="video" autoplay></video>
+    <canvas ref="canvas"></canvas>
   </div>
 </template>
 
 <script>
+import * as posenet from "@tensorflow-models/posenet";
+import * as utils from "../utils";
+
 export default {
   data() {
     return {
@@ -13,10 +18,10 @@ export default {
     };
   },
   created() {
-    console.log("Starting connection to websocet server");
+    console.log("Starting connection to websocket server");
     console.log(this.$route.params.id);
     this.connection = new WebSocket(
-      "ws://192.168.1.244:4000/" + this.$route.params.id
+      "ws://50.100.180.37:4000/" + this.$route.params.id
     );
 
     this.connection.onopen = function (event) {
@@ -28,7 +33,12 @@ export default {
       console.log(event); // When we recieve a message
     };
   },
-  mounted() {},
+  async mounted() {
+    this.ctx = this.$refs.canvas.getContext("2d");
+    this.net = await posenet.load();
+    this.streamPromise = await this.initWebcamStream();
+    this.detectPose();
+  },
   methods: {
     sendMessage() {
       console.log(this.connection);
@@ -53,6 +63,82 @@ export default {
       }
       return "";
     },
+    initWebcamStream() {
+      // if the browser supports mediaDevices.getUserMedia API
+      if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
+        return navigator.mediaDevices
+          .getUserMedia({
+            audio: false, // don't capture audio
+            video: { facingMode: "environment" }, // use the rear camera if there is
+          })
+          .then((stream) => {
+            // set <video> source as the webcam input
+            let video = this.$refs.video;
+            try {
+              video.srcObject = stream;
+            } catch (error) {
+              // support older browsers
+              video.src = URL.createObjectURL(stream);
+            }
+            return new Promise((resolve) => {
+              // when video is loaded
+              video.onloadedmetadata = () => {
+                // calculate the video ratio
+                this.videoRatio = video.offsetHeight / video.offsetWidth;
+                // add event listener on resize to reset the <video> and <canvas> sizes
+                window.addEventListener("resize", this.setResultSize);
+                // set the initial size
+                this.setResultSize();
+                resolve();
+              };
+            });
+          })
+          .catch((error) => {
+            console.log("failed to initialize webcam stream", error);
+            throw error;
+          });
+      } else {
+        return Promise.reject(
+          new Error(
+            "Your browser does not support mediaDevices.getUserMedia API"
+          )
+        );
+      }
+    },
+    setResultSize () {
+      // get the current browser window size
+      let clientWidth = document.documentElement.clientWidth
+      // set max width as 600
+      this.resultWidth = Math.min(600, clientWidth)
+      // set the height according to the video ratio
+      this.resultHeight = this.resultWidth * this.videoRatio
+      // set <video> width and height
+      /*
+        Doesn't use vue binding :width and :height,
+          because the initial value of resultWidth and resultHeight
+          will affect the ratio got from the initWebcamStream()
+      */
+      let video = this.$refs.video
+      video.width = this.resultWidth
+      video.height = this.resultHeight
+    },
+    timeout(ms){
+        return new Promise(resolve => setTimeout(resolve, ms));
+    },
+    async detectPose() {
+        const imageScaleFactor = 0.5;
+        const flipHorizontal = false;
+        const outputStride = 16;
+        this.pose = await this.net.estimateSinglePose(this.$refs.video, imageScaleFactor, flipHorizontal, outputStride) 
+        console.log(this.pose);
+        utils.drawSkeleton(this.pose.keypoints, 0.3, this.ctx);
+        //this.renderPose();
+        // await this.timeout(1000/20);
+        // return this.detectPose();
+        requestAnimationFrame(() => {
+            this.detectPose();
+        })
+    }
   },
 };
 </script>

src/components/Home.vue

@@ -15,7 +15,7 @@ export default {
   },
   methods: {
     createRoom() {
-      this.axios.get("http://localhost:4000/rooms/add").then((res) => {
+      this.axios.get("http://50.100.180.37:4000/rooms/add").then((res) => {
         let id = res.data._id;
         let playerId = res.data.playerId;
         this.setCookie("playerId", playerId, 1);
@@ -24,7 +24,7 @@ export default {
     },
     joinRoom() {
       console.log(this.room);
-      this.axios.post("http://localhost:4000/rooms/join", this.room).then((res) => {
+      this.axios.post("http://50.100.180.37:4000/rooms/join", this.room).then((res) => {
         let playerId = res.data._id;
         this.setCookie("playerId", playerId, 1);
         this.$router.push({path: `game/${this.room.id}`});

src/utils.js

@@ -0,0 +1,44 @@
+import * as posenet from "@tensorflow-models/posenet";
+
+const color = "aqua";
+const lineWidth = 2;
+
+
+function toTuple({x, y}){
+    return [x, y];
+}
+
+export function drawPoint(ctx, x, y, r, color){
+    ctx.beginPath();
+    ctx.arc(x, y, r, 0, 2*Math.PI);
+    ctx.fillStyle = color;
+    ctx.fill;
+}
+
+export function drawSegment([ax, ay], [bx, by], color, scale, ctx){
+    ctx.beginPath();
+    ctx.moveTo(ax*scale, ay*scale);
+    ctx.lineTo(bx*scale, by*scale);
+    ctx.lineWidth = lineWidth;
+    ctx.strokeStyle = color;
+    ctx.stroke();
+}
+
+export function drawSkeleton(keypoints, minConfidence, ctx, scale=1){
+    
+    ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height)
+    const adjacentKeyPoints = posenet.getAdjacentKeyPoints(keypoints, minConfidence);
+    adjacentKeyPoints.forEach(keypoints => {
+        drawSegment(toTuple(keypoints[0].position), toTuple(keypoints[1].position), color, scale, ctx);
+    })
+}
+
+export function drawKeypoints(keypoints, minConfidence, ctx, scale = 1){
+    keypoints.forEach(keypoints => {
+        if (keypoints.score < minConfidence){
+            return; 
+        }
+        const {x, y} = keypoints.position; 
+        drawPoint(ctx, x*scale, y*scale, 3, color);
+    })
+}