Postprocess object detection predictions with tfjs-tflite

Special Interest Groups TF.js
,
#1

I recently created an object detection example using tfjs-tflite, which uses the ObjectDetector class to load and use the Object Detector.

Now I wanted to create an object detection without using the ObjectDetector class. I managed to load the model into memory and to prepare the image to make predictions by following the ‘Test model runner’ example, but I’m having problems postprocessing the predictions since the dataSync method that is used in the CodePen example throws an error.

index.js:20 Uncaught (in promise) TypeError: result.dataSync is not a function
    at detect (index.js:20)

Without the dataSync() method I’m getting the following output:

{TFLite_Detection_PostProcess: e, TFLite_Detection_PostProcess:1: e, TFLite_Detection_PostProcess:2: e, TFLite_Detection_PostProcess:3: e}
TFLite_Detection_PostProcess: e {kept: false, isDisposedInternal: false, shape: Array(3), dtype: 'float32', size: 40, …}
TFLite_Detection_PostProcess:1: e {kept: false, isDisposedInternal: false, shape: Array(2), dtype: 'float32', size: 10, …}
TFLite_Detection_PostProcess:2: e {kept: false, isDisposedInternal: false, shape: Array(2), dtype: 'float32', size: 10, …}
TFLite_Detection_PostProcess:3: e {kept: false, isDisposedInternal: false, shape: Array(1), dtype: 'float32', size: 1, …}
[[Prototype]]: Object

Code:
index.js:

const img = document.querySelector("img");
const resultEle = document.querySelector(`.result`);
let objectDetector;

/** Detect objects in image. */
async function detect() {
    resultEle.textContent = "Loading...";
    if (!objectDetector) {
        objectDetector = await tflite.loadTFLiteModel(
            "https://tfhub.dev/tensorflow/lite-model/ssd_mobilenet_v1/1/metadata/2?lite-format=tflite"
        );
    }
    
    const start = Date.now();
    let input = tf.image.resizeBilinear(tf.browser.fromPixels(img), [300, 300]);
    input = tf.cast(tf.sub(tf.div(tf.expandDims(input), 127.5), 1), 'int32');
    
    // Run the inference and get the output tensors.
    let result = objectDetector.predict(input);
    console.log(result)
    const latency = Date.now() - start;
    renderDetectionResult(result);
    resultEle.textContent = `Latency: ${latency}ms`;
}

/** Render detection results. */
function renderDetectionResult(result) {
    const boxesContainer = document.querySelector(".boxes-container");
    boxesContainer.innerHTML = "";
    for (let i = 0; i < result.length; i++) {
        const curObject = result[i];
        const boundingBox = curObject.boundingBox;
        const name = curObject.classes[0].className;
        const score = curObject.classes[0].probability;

        if (score > 0.5) {
            const boxContainer = createDetectionResultBox(
                boundingBox.originX,
                boundingBox.originY,
                boundingBox.width,
                boundingBox.height,
                name,
                score
            );
            boxesContainer.appendChild(boxContainer);
        } 
    }
}

/** Create a single detection result box. */
function createDetectionResultBox(left, top, width, height, name, score) {
    const container = document.createElement("div");
    container.classList.add("box-container");

    const box = document.createElement("div");
    box.classList.add("box");
    container.appendChild(box);

    const label = document.createElement("div");
    label.classList.add("label");
    label.textContent = `${name} (${score.toFixed(2)})`;
    container.appendChild(label);

    container.style.left = `${left - 1}px`;
    container.style.top = `${top - 1}px`;
    box.style.width = `${width + 1}px`;
    box.style.height = `${height + 1}px`;

    return container;
}

document.querySelector(".btn").addEventListener("click", () => {
    detect();
});

index.html

<!DOCTYPE html>
<html lang="en">

<head>
    <meta charset="UTF-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <title>TFLITE Web API Example</title>
    <link rel="stylesheet" href="style.css">
</head>

<body>
    <h1>TFLITE Web API Object Detection Example</h1>
    <div class="img-container">
        <img src="https://storage.googleapis.com/tfweb/demos/static/obj_detection.jpeg" crossorigin="anonymous" />
        <div class="boxes-container"></div>
    </div>
    <div class="btn">Detect</div>
    <div class="result"></div>
    <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-core"></script>
    <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-cpu"></script>
    <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-tflite@0.0.1-alpha.6/dist/tf-tflite.min.js"></script>
    <script src="index.js"></script>
</body>

</html>

Any help is highly appreciated. Kind regards,
Gilbert Tanner

#3

@Jason can help here

1 Like
#4

So when you predict on the model that operation is async so you need to await to that to finish before you attempt to access the result object which may not have resolved yet.

For example:

let result = await objectDetector.predict(input);
let data = await result.data();
console.log(data);
2 Likes
#5

Thanks for the reply @Jason. I couldn’t get it to work by calling the data method on the result directly as this gave me an error but it worked, when calling it on the individual results:

let boxes = Array.from(await result["TFLite_Detection_PostProcess"].data());
let classes = Array.from(await result["TFLite_Detection_PostProcess:1"].data())
let scores = Array.from(await result["TFLite_Detection_PostProcess:2"].data())
let n = Array.from(await result["TFLite_Detection_PostProcess:3"].data())

The complete code is available on my Github. If you find anything that can be improved please let me know.

Kind regards,
Gilbert Tanner

#6

Glad you got it working with the await! Indeed the function can be called on Tensors so as long as what is returned is a Tensor then it should be able to call data on that. See TF Tensor API here: