How do i input a 2D-matrix into a quantified Tensorflow Lite model on microcontrollers?

Hello Tensorflow Team,

i want to put acceleration and gyroscope sensordata into a full int8 quantified neural network. All this should run on an ESP32 microcontroller and i use the Arduino IDE 2.0.2. The data the sensor gives me is a matrix with the shape (6,128). I normalize and then quantify the matrix like it is shown in the hello world example. But now i have the problem that i don’t know on how to put my matrix into the network to make predictions. As i try to do this it throws an error. I will link my code and error below. It is still work in progress.
Thanks in advance.

The Code:

// Tensorflow imports
#include <TensorFlowLite_ESP32.h>
#include "tensorflow/lite/micro/all_ops_resolver.h"
#include "tensorflow/lite/micro/micro_error_reporter.h"
#include "tensorflow/lite/micro/micro_interpreter.h"
#include "tensorflow/lite/micro/system_setup.h"
#include "tensorflow/lite/schema/schema_generated.h"
#include "ESP32_Model.h"

// MPU-6050 imports
#include <Adafruit_MPU6050.h>
#include <Adafruit_Sensor.h>
#include <Wire.h>

Adafruit_MPU6050 mpu;

namespace {
  tflite::ErrorReporter* error_reporter = nullptr;
  const tflite::Model* model = nullptr;
  tflite::MicroInterpreter* interpreter = nullptr;
  TfLiteTensor* input = nullptr;
  TfLiteTensor* output = nullptr;

// Create Memory arena for calculations

  constexpr int kTensorArenaSize = 8192;
  uint8_t tensor_arena[kTensorArenaSize];

}

void setup(){

  // Set up the MPU-6050 --------------------------------------------
  Serial.begin(9600);
  while (!Serial){
    delay(10);
  }

  if (!mpu.begin()) {
    while (1) {
      delay(10);
    }
  }

  mpu.setAccelerometerRange(MPU6050_RANGE_2_G);
  mpu.setGyroRange(MPU6050_RANGE_250_DEG);
  mpu.setFilterBandwidth(MPU6050_BAND_260_HZ);

  // Set up Tensorflow ----------------------------------------------
  static tflite::MicroErrorReporter micro_error_reporter;
  error_reporter = &micro_error_reporter; 

  // Map the model into a usable data structure. This doesn't involve any
  // copying or parsing, it's a very lightweight operation.
  model = tflite::GetModel(ESP32_Model);
  if (model->version() != TFLITE_SCHEMA_VERSION) {
    TF_LITE_REPORT_ERROR(error_reporter,
                         "Model provided is schema version %d not equal "
                         "to supported version %d.",
                         model->version(), TFLITE_SCHEMA_VERSION);
    return;
  }

  // This pulls in all the operation implementations we need.
  // NOLINTNEXTLINE(runtime-global-variables)
  static tflite::AllOpsResolver resolver;

  // Build an interpreter to run the model with.
  static tflite::MicroInterpreter static_interpreter(
      model, resolver, tensor_arena, kTensorArenaSize, error_reporter);
  interpreter = &static_interpreter;

  // Allocate memory from the tensor_arena for the model's tensors.
  TfLiteStatus allocate_status = interpreter->AllocateTensors();
  if (allocate_status != kTfLiteOk) {
    TF_LITE_REPORT_ERROR(error_reporter, "AllocateTensors() failed");
    return;
  }

  // Obtain pointers to the model's input and output tensors.
  input = interpreter->input(2);
  output = interpreter->output(2);

}

float Accx = 0.0;
float Accy = 0.0;
float Accz = 0.0;
float Gyrx = 0.0;
float Gyry = 0.0;
float Gyrz = 0.0;
float Temp = 0.0;

sensors_event_t Acc, Gyr, Temps;

float Input_Matrix_float[6][128];
float Output_Matrix_float[6][128];
float Input_Matrix_norm[6][128];
int8_t Input_Matrix_quant[6][128];
int8_t Output_Matrix_quant[6][128];

int Index = 0;
float Max = 0.0;
float Min = 0.0;
int counter = 0;
int Pos_Min = 0;
int Pos_Max = 0;
int8_t Storage = 0;
float Abs_Sum = 0.0;
float MAE = 0.0;

void loop(){

  mpu.getEvent(&Acc,&Gyr,&Temps);

  Input_Matrix_float[0][Index] = Acc.acceleration.x;
  Input_Matrix_float[1][Index] = Acc.acceleration.y;
  Input_Matrix_float[2][Index] = Acc.acceleration.z;

  Input_Matrix_float[3][Index] = Gyr.acceleration.x;
  Input_Matrix_float[4][Index] = Gyr.acceleration.y;
  Input_Matrix_float[5][Index] = Gyr.acceleration.z;

  Index += 1;

  if (Index==128){

    Serial.print("\n");
    Serial.print("\n");
    Serial.print("\n");
    Serial.print("Input Matrix float\n");
    counter = 0;

    for (int i=0; i<128; i++){
        for (int k=0; k<6; k++){
          Serial.print(Input_Matrix_float[k][i]);
          Serial.print(" ");
      }
      Serial.print(counter);
      Serial.print("\n");
      counter += 1;
    }

    for (int i=0; i<128; i++){
      for (int k=0; k<6; k++){
        if (Max < Input_Matrix_float[k][i]){
          Max = Input_Matrix_float[k][i];
          Pos_Max = i;
        }

        if (Min > Input_Matrix_float[k][i]){
          Min = Input_Matrix_float[k][i];
          Pos_Min = i;

        }
      }

    }

    Serial.print("Min Wert\n");
    Serial.print(Min);
    Serial.print(" ");
    Serial.print(Pos_Min);
    Serial.print("\nMax Wert\n");
    Serial.print(Max);
    Serial.print(" ");
    Serial.print(Pos_Max);
    Serial.print("\n");

    counter = 0;
    Serial.print("Input Matrix norm\n");
    for (int i=0; i<128; i++){
      for (int k=0; k<6; k++){
        Input_Matrix_norm[k][i] = (2*(Input_Matrix_float[k][i]-Min)/(Max-Min))-1;
        Serial.print(Input_Matrix_norm[k][i]);
        Serial.print(" ");
      }
      Serial.print(counter);
      Serial.print("\n");
      counter += 1;
    }

    counter = 0;
    Serial.print("Input Matrix quant\n");
    for (int i=0; i<128; i++){
        for (int k=0; k<6; k++){
          Input_Matrix_quant[k][i] = int8_t(Input_Matrix_float[k][i] / input->params.scale + input->params.zero_point);
          Serial.print(Input_Matrix_quant[k][i]);
          Serial.print(" ");
      }
      Serial.print(counter);
      Serial.print("\n");
      counter += 1;
    }
    
    //????? 
    input->data.int8[0] = Input_Matrix_quant;

    TfLiteStatus invoke_status = interpreter->Invoke();

    if (invoke_status != kTfLiteOk) {
      TF_LITE_REPORT_ERROR(error_reporter, "Invoke failed on x: %f\n",
                         static_cast<double>(Input_Matrix_quant));
    return;

    counter = 0;
    Serial.print("Output Matrix quant\n");
    for (int i=0; i<128; i++){
      for (int k=0; k<6; k++){
        Output_Matrix_quant[k][i] = output->data.int8[k][i];
        Serial.print(Output_Matrix_quant[k][i]);
        Serial.print(" ");
      }
      Serial.print(counter);
      Serial.print("\n");
    }

    counter = 0;
    Serial.print("Output Matrix float\n");
    for (int i=0; i<128; i++){
      for (int k=0; k<6; k++){
        Output_Matrix_float[k][i] = (Output_Matrix_quant[k][i]-params.zero_point)*output->params.scale;
        Serial.print(Output_Matrix_float[k][i]);
        Serial.print(" ");
      }
      Serial.print(counter);
      Serial.print("\n");
    }

    for (int i=0; i<128; i++){
      for (int k=0; k<6; k++){
         Abs_Sum += sqrt(sq(Output_Matrix_float[k][i]));
      }
    }

    MAE = Abs_Sum/(6*128);

    Serial.print("MAE ist: ");
    Serial.print(MAE);
    Serial.print("\n");

    // Set all Arrays and Vars to 0
    for (int i=0; i<128; i++){
        for (int k=0; k<6; k++){
          Input_Matrix_float[k][i] = 0.0;
      }
    }

    for (int i=0; i<128; i++){
        for (int k=0; k<6; k++){
          Input_Matrix_norm[k][i] = 0.0;
      }
    }

    for (int i=0; i<128; i++){
        for (int k=0; k<6; k++){
          Input_Matrix_quant[k][i] = 0;
      }
    }

    for (int i=0; i<128; i++){
        for (int k=0; k<6; k++){
          Output_Matrix_quant[k][i] = 0;
      }
    }

    for (int i=0; i<128; i++){
        for (int k=0; k<6; k++){
          Output_Matrix_float[k][i] = 0.0;
      }
    }

    Min = 0.0;
    Max = 0.0;
    Pos_Min = 0;
    Pos_Max = 0;
    counter = 0;
    Abs_Sum = 0.0;
    MAE = 0.0;
    Index = 0;

  }

delay(4);

}

The Error:

C:\Users\PaulR\Documents\Arduino\ESP32_Model_with_MPU6050\ESP32_Model_with_MPU6050.ino: In function 'void loop()':
C:\Users\PaulR\Documents\Arduino\ESP32_Model_with_MPU6050\ESP32_Model_with_MPU6050.ino:197:27: error: invalid conversion from 'int8_t (*)[128]' {aka 'signed char (*)[128]'} to 'int8_t' {aka 'signed char'} [-fpermissive]
     input->data.int8[0] = Input_Matrix_quant;
                           ^~~~~~~~~~~~~~~~~~
In file included from c:\Users\PaulR\Documents\Arduino\libraries\TensorFlowLite_ESP32\src/tensorflow/lite/micro/micro_mutable_op_resolver.h:22,
                 from c:\Users\PaulR\Documents\Arduino\libraries\TensorFlowLite_ESP32\src/tensorflow/lite/micro/all_ops_resolver.h:19,
                 from C:\Users\PaulR\Documents\Arduino\ESP32_Model_with_MPU6050\ESP32_Model_with_MPU6050.ino:3:
C:\Users\PaulR\Documents\Arduino\ESP32_Model_with_MPU6050\ESP32_Model_with_MPU6050.ino:203:64: error: invalid static_cast from type 'int8_t [6][128]' {aka 'signed char [6][128]'} to type 'double'
                          static_cast<double>(Input_Matrix_quant));
                                                                ^
c:\Users\PaulR\Documents\Arduino\libraries\TensorFlowLite_ESP32\src/tensorflow/lite/core/api/error_reporter.h:53:59: note: in definition of macro 'TF_LITE_REPORT_ERROR'
     static_cast<tflite::ErrorReporter*>(reporter)->Report(__VA_ARGS__); \
                                                           ^~~~~~~~~~~
C:\Users\PaulR\Documents\Arduino\ESP32_Model_with_MPU6050\ESP32_Model_with_MPU6050.ino:210:59: error: invalid types 'int8_t {aka signed char}[int]' for array subscript
         Output_Matrix_quant[k][i] = output->data.int8[k][i];
                                                           ^
C:\Users\PaulR\Documents\Arduino\ESP32_Model_with_MPU6050\ESP32_Model_with_MPU6050.ino:222:64: error: 'params' was not declared in this scope
         Output_Matrix_float[k][i] = (Output_Matrix_quant[k][i]-params.zero_point)*output->params.scale;
                                                                ^~~~~~
C:\Users\PaulR\Documents\Arduino\ESP32_Model_with_MPU6050\ESP32_Model_with_MPU6050.ino:286:1: error: expected '}' at end of input
 }
 ^
C:\Users\PaulR\Documents\Arduino\ESP32_Model_with_MPU6050\ESP32_Model_with_MPU6050.ino:112:12: note: to match this '{'
 void loop(){
            ^

exit status 1

Compilation error: invalid conversion from 'int8_t (*)[128]' {aka 'signed char (*)[128]'} to 'int8_t' {aka 'signed char'} [-fpermissive]

Got any Solution?

I am somewhat working on the same problem. I want to to Push a 2D array into my model for the prediction. The input required shape is {1x10x6}. Could you please help me with this?

Thanks in Advance.
Regards,
Jatin

The code:
/**

  • Test sinewave neural network model
  • Author: Pete Warden
  • Modified by: Shawn Hymel
  • Date: March 11, 2020
  • Copyright 2019 The TensorFlow Authors. All Rights Reserved.
  • Licensed under the Apache License, Version 2.0 (the “License”);
  • you may not use this file except in compliance with the License.
  • You may obtain a copy of the License at
  • http://www.apache.org/licenses/LICENSE-2.0
  • Unless required by applicable law or agreed to in writing, software
  • distributed under the License is distributed on an “AS IS” BASIS,
  • WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  • See the License for the specific language governing permissions and
  • limitations under the License.
    */

#include <Arduino_LSM6DSOX.h>

unsigned long current_time;
float time_difference;

int N = 5; // since, average is over consecutive 5 readings

float IMU_data[7]; //an array for IMU values, (1,2,3)=> accelerometer values and (4,5,6)=> gyroscope values_
//
float IMU_6N_data[7][6]; //a 2D array to save consecutive N IMU values // 6 because it contains 3 accelerometer values and 3 gyroscope values
//

// Import TensorFlow stuff
#include <TensorFlowLite.h>
#include <tensorflow/lite/micro/all_ops_resolver.h>
#include <tensorflow/lite/micro/micro_error_reporter.h>
#include <tensorflow/lite/micro/micro_interpreter.h>
#include <tensorflow/lite/schema/schema_generated.h>
#include <tensorflow/lite/version.h>

// Our model
#include “c_model_v2.h”

// Figure out what’s going on in our model
#define DEBUG 1

// TFLite globals, used for compatibility with Arduino-style sketches
namespace {
tflite::MicroErrorReporter error_reporter = nullptr;
const tflite::Model
model = nullptr;
tflite::MicroInterpreter* interpreter = nullptr;
TfLiteTensor* model_input = nullptr;
TfLiteTensor* model_output = nullptr;

// Create an area of memory to use for input, output, and other TensorFlow
// arrays. You’ll need to adjust this by combiling, running, and looking
// for errors.
constexpr int kTensorArenaSize = 5 * 1024;
uint8_t tensor_arena[kTensorArenaSize];
} // namespace

// array to map gesture index to a name
const char* GESTURES[] = {
“NO motion”,
“Left Swing”,
“Right Swing”,
“Adduction”,
“Flexion”
};

#define NUM_GESTURES 5

int N_samples = 10; //a variable for sample window
float Samples[11][6] = {0};

void setup()
{

// Wait for Serial to connect
#if DEBUG
while(!Serial);
#endif

// Set up logging (will report to Serial, even within TFLite functions)
static tflite::MicroErrorReporter micro_error_reporter;
error_reporter = &micro_error_reporter;

// Map the model into a usable data structure
model = tflite::GetModel(c_model_v2);
if (model->version() != TFLITE_SCHEMA_VERSION)
{
Serial.println(“Model schema mismatch!”);
while(1);
}

// pull in all the TFLM ops, you can remove this line and
// only pull in the TFLM ops you need, if would like to reduce
// the compiled size of the sketch.
tflite::AllOpsResolver tflOpsResolver;

// Build an interpreter to run the model
interpreter = new tflite::MicroInterpreter(model, tflOpsResolver, tensor_arena, kTensorArenaSize, error_reporter);

// Allocate memory for the model’s input and output tensors
interpreter->AllocateTensors();

  // Assign model input and output buffers (tensors) to pointers

model_input = interpreter->input(0);
model_output = interpreter->output(0);

// Get information about the memory area to use for the model’s input
// Supported data types:
// https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/c/common.h#L226
/*
#if DEBUG
Serial.print("Number of dimensions: ");
Serial.println(model_input->dims->size);
Serial.print("Dim 1 size: ");
Serial.println(model_input->dims->data[0]);
Serial.print("Dim 2 size: ");
Serial.println(model_input->dims->data[1]);
Serial.print("Dim 3 size: ");
Serial.println(model_input->dims->data[2]);
Serial.print("Input type: ");
Serial.println(model_input->type);
#endif
*/
}

void loop()
{

float IMU_avg[7] = {0.0,0.0,0.0,0.0,0.0,0.0,0.0}; // an array for average accelerometer and gyroscope values
float Samples[11][6] = {0};

for(int k =1; k<=N_samples; k++)
{

//////////////////////////////////////////////////    reading accelerometer and gyroscope values  /////////////////////////////////////////////////////////////////   

for(int j=1; j<=N; j++)                                                                        //loop for saving 5 consecutive values of accelerometer into a 2D array
{
  if (IMU.accelerationAvailable() & IMU.gyroscopeAvailable())                                    //Get accelerometer values and gyroscople values together
  {
    IMU.readAcceleration(IMU_data[1], IMU_data[2], IMU_data[3]);
    IMU.readGyroscope(IMU_data[4], IMU_data[5], IMU_data[6]);   

    for(int i=1; i<=3; i++)
    {
      IMU_6N_data[i][j] = IMU_data[i];                                                      //Save 5 consecutive accelerometer values into IMU_6N_data 
      IMU_6N_data[i+3][j] = IMU_data[i+3];                                                  //Save 5 consecutive gyroscope values into IMU_6N_data
    }
  }
}

////////////////////////////////////////////////    reading accelerometer and gyroscop value: complete ///////////////////////////////////////////////////////

//////////////////////////////////////////////////              average filter                        ////////////////////////////////////////// 

for(int i =1; i<4; i++)                                          //loop for taking an average of the accelerometer  and gyroscope values
{
  for(int j=1; j<N; j++)
  {
    IMU_avg[i] = IMU_avg[i] + ((IMU_6N_data[i][j])/(float)N);                                //statement taking average of Accelerometer value
    IMU_avg[i+3] = IMU_avg[i+3] + ((IMU_6N_data[i+3][j])/(float)N);      //*                      //statement taking average of Gyroscope values
   }
   
   Samples[k][i] = IMU_avg[i];
   Samples[k][i+3] = IMU_avg[i+3];
 }

 ///////////////////////////////////////////////             average flter: complete                   ///////////////////////////////////////////////

}
/*
///////////////////////////////////////////////// Pushing Sample Window to Model /////////////////////////////////////////////////////////////////
for(int k=0; k<N_samples; k++)
{
for(int i=0; i<6; i++)
{
model_input->data.f[i] = Samples[k][i];
}
}
///////////////////////////////////////////////////////////Complete////////////////////////////////////////////////////////////////////////////////
*/

model_input->data.f[0] = Samples;

// Run inference
TfLiteStatus invoke_status = interpreter->Invoke();
if (invoke_status != kTfLiteOk) {
Serial.println(“Invoke failed!”);
return;
}

for (int i = 0; i < NUM_GESTURES; i++)
{
Serial.print(GESTURES[i]);
Serial.print(": ");
Serial.println(model_output->data.f[i], 6);
Serial.println();
}
}

Hello, @Jatin_kadge and @Palettenbrett, have you solved this issue, i am also facing the same.

Regards,
Vasanth P