open website on phone browser, browse local image
press predict, start calculating prediction result
predicted a soccer ball
predicted a basketball
open browser on desktop -> browser a goose image ->
copy paste image url -> predict
mobileNet distinguish duck from goose
import React, { useState, useEffect } from 'react';
import * as tf from '@tensorflow/tfjs'
import imagenet_label from './imagenet_label.json'
import '../pages/Home.css';
import domtoimage from 'dom-to-image';
export default function MobileNet(props) {
const [model, setModel] = useState(null)
const [model_predictions, setModel_predictions] = useState(null)
useEffect(() => { loadModel() }, [props.predict]);
const loadModel = async () => {
const m = await tf.loadLayersModel('/assets/model.json')
setModel(m)
if (props.predict > 0) {
setModel_predictions(null)
//convert media to png before processing image
domtoimage.toPng(props.image)
.then(function (dataUrl) {
var img = new Image();
img.src = dataUrl;
//wait for new png image
var timer = setInterval(function () {
if (img.src) {
processImage(img)
clearInterval(timer)
}
}, 100)
})
.catch(function (error) {
alert("error converting image to png");
});
}
}
const processImage = async (image) => {
const pixels = tf.browser.fromPixels(image).resizeNearestNeighbor([224, 224]).toFloat()
const offset = tf.scalar(127.5)
const preprocessed_image = pixels.sub(offset).div(offset).expandDims()
const predictions = await model.predict(preprocessed_image).data()
//predictions is Float32Array(1000), find top5 value with their index
let top5 = []
for (let i = 0; i < 5; i++) {
const imax = predictions.indexOf(Math.max(...predictions));
const key = imagenet_label[imax.toString()]
const value = predictions[imax].toExponential(2).toString()
top5.push(<tr key={key}>
<td>{key}</td>
<td>{value}</td>
</tr>)
predictions[imax] = 0
}
/*top5
Array(5)
0: Object { "bald eagle, American eagle, Haliaeetus leucocephalus": "9.60e-1" }
1: Object { kite: "3.93e-2" }
2: Object { "black grouse": "9.76e-5" }
3: Object { vulture: "8.26e-5" }
4: Object { "black stork, Ciconia nigra": "7.81e-5" }
*/
setModel_predictions(top5)
}
return (
<div>
{props.predict > 0 && !model_predictions ? <span>Calculating...</span> : null}
{model_predictions ?
<table>
<thead><tr><th>Object</th><th>Probability</th></tr></thead>
<tbody>{model_predictions}</tbody>
</table> : null}
</div>
)
}
------------------------------
//home.tsx
import {
IonContent, IonHeader, IonPage, IonTitle, IonToolbar,
IonCard, IonCardContent, IonCardHeader, IonCardSubtitle, IonCardTitle,
IonItem, IonIcon, IonLabel, IonButton, IonInput
} from '@ionic/react';
import React, { useState } from 'react';
import './Home.css';
import MobileNet from '../components/tensorflow_mobileNet'
const Home: React.FC = () => {
const [predictClick, setPredictClick] = useState(0);
const [imageUrl, setImageUrl] = useState('https://images.theconversation.com/files/308043/original/file-20191220-11924-iakhpb.jpeg?ixlib=rb-1.1.0&q=45&auto=format&w=754&fit=clip')
const browseImg = (e: any) => {
const fileURL = URL.createObjectURL(e.target.files[0])
//get url where uploaded image is temporarily stored
//blob:http://localhost:3000/11c28a94-d9df-43e1-ab6c-2185e1449eb6
setImageUrl(fileURL)
}
return (
<IonPage>
<IonHeader>
<IonToolbar>
<IonTitle>
Predict Image with MobileNet
</IonTitle>
</IonToolbar>
</IonHeader>
<IonContent>
<IonCard>
<IonCardHeader>
<IonCardTitle>
MobileNet Input Image
</IonCardTitle>
</IonCardHeader>
<IonItem lines="none">
<IonButton fill="outline"
onClick={() => document.getElementById('input-image-browser')?.click()}>
Browse Local Image</IonButton>
<input hidden type="file" accept="image/*" id="input-image-browser"
onChange={(e) => browseImg(e)}></input>
</IonItem>
<IonItem >
<IonLabel slot="start">Paste Web Image URL</IonLabel>
<IonInput type="url" placeholder="https://image.com/image.jpg"
onIonChange={(e) => setImageUrl(e.detail.value!)}></IonInput>
</IonItem>
<IonItem lines="none">
<img id='test_image' src={imageUrl} alt="Input Image"></img>
</IonItem>
<IonItem lines="none">
<IonButton fill="outline" onClick={() => setPredictClick(predictClick + 1)}>Predict</IonButton>
</IonItem>
<IonCardHeader>
<IonCardTitle>
MobileNet Prediction Result
</IonCardTitle>
</IonCardHeader>
<IonItem lines="none">
<MobileNet image={document.getElementById('test_image')} predict={predictClick} />
</IonItem>
</IonCard>
</IonContent>
</IonPage>
);
};
export default Home;
------------------------------
//package.json
{
"name": "ionic-keras",
"version": "0.0.1",
"private": true,
"dependencies": {
"@capacitor/core": "2.4.6",
"@ionic/react": "^5.0.7",
"@ionic/react-router": "^5.0.7",
"@tensorflow/tfjs": "^2.8.4",
"@testing-library/jest-dom": "^4.2.4",
"@testing-library/react": "^9.4.0",
"@testing-library/user-event": "^8.0.3",
"@types/jest": "^24.0.25",
"@types/node": "^12.12.24",
"@types/react": "^16.9.17",
"@types/react-dom": "^16.9.4",
"@types/react-router": "^5.1.4",
"@types/react-router-dom": "^5.1.3",
"dom-to-image": "^2.6.0",
"ionicons": "^5.0.0",
"react": "^16.13.0",
"react-dom": "^16.13.0",
"react-router": "^5.1.2",
"react-router-dom": "^5.1.2",
"react-scripts": "3.4.0",
"typescript": "3.8.3"
},
"scripts": {
"start": "react-scripts start",
"build": "react-scripts build",
"test": "react-scripts test",
"eject": "react-scripts eject"
},
"eslintConfig": {
"extends": "react-app"
},
"browserslist": {
"production": [
">0.2%",
"not dead",
"not op_mini all"
],
"development": [
"last 1 chrome version",
"last 1 firefox version",
"last 1 safari version"
]
},
"devDependencies": {
"@capacitor/cli": "2.4.6"
},
"description": "An Ionic project"
}
reference:
No comments:
Post a Comment