Finishing stock counting

5-retail/lessons/2-check-stock-device/README.md

@@ -97,7 +97,7 @@ In the image above, 4 cans of tomato paste were detected. In the results a red s
 
 ✅ Open the predictions in Custom Vision and check out the bounding boxes.
 
-Bounding boxes are defined with 4 values - top, left, height and width. These values are on a scale of 0-1, representing the positions as a percentage of the size of the image.
+Bounding boxes are defined with 4 values - top, left, height and width. These values are on a scale of 0-1, representing the positions as a percentage of the size of the image. The origin (the 0,0 position) is the top left of the image, so the top value is the distance from the top, and the bottom of the bounding box is the top plus the height.
 
 ![A bounding box around a can of tomato paste](../../../images/bounding-box.png)
 

5-retail/lessons/2-check-stock-device/code-count/wio-terminal/fruit-quality-detector/src/main.cpp

@@ -5,6 +5,7 @@
 #include "SD/Seeed_SD.h"
 #include <Seeed_FS.h>
 #include <SPI.h>
+#include <vector>
 #include <WiFiClientSecure.h>
 
 #include "config.h"
@@ -57,7 +58,99 @@ void setup()
     pinMode(WIO_KEY_C, INPUT_PULLUP);
 }
 
-const float threshold = 0.3f;
+const float threshold = 0.0f;
+const float overlap_threshold = 0.20f;
+
+struct Point {
+    float x, y;
+};
+
+struct Rect {
+    Point topLeft, bottomRight;
+};
+
+float area(Rect rect)
+{
+    return abs(rect.bottomRight.x - rect.topLeft.x) * abs(rect.bottomRight.y - rect.topLeft.y);
+}
+    
+float overlappingArea(Rect rect1, Rect rect2)
+{
+    float left = max(rect1.topLeft.x, rect2.topLeft.x);
+    float right = min(rect1.bottomRight.x, rect2.bottomRight.x);
+    float top = max(rect1.topLeft.y, rect2.topLeft.y);
+    float bottom = min(rect1.bottomRight.y, rect2.bottomRight.y);
+
+
+    if ( right > left && bottom > top )
+    {
+        return (right-left)*(bottom-top);
+    }
+    
+    return 0.0f;
+}
+
+Rect rectFromBoundingBox(JsonVariant prediction)
+{
+    JsonObject bounding_box = prediction["boundingBox"].as<JsonObject>();
+
+    float left = bounding_box["left"].as<float>();
+    float top = bounding_box["top"].as<float>();
+    float width = bounding_box["width"].as<float>();
+    float height = bounding_box["height"].as<float>();
+
+    Point topLeft = {left, top};
+    Point bottomRight = {left + width, top + height};
+
+    return {topLeft, bottomRight};
+}
+
+void processPredictions(std::vector<JsonVariant> &predictions)
+{
+    std::vector<JsonVariant> passed_predictions;
+
+    for (int i = 0; i < predictions.size(); ++i)
+    {
+        Rect prediction_1_rect = rectFromBoundingBox(predictions[i]);
+        float prediction_1_area = area(prediction_1_rect);
+        bool passed = true;
+
+        for (int j = i + 1; j < predictions.size(); ++j)
+        {
+            Rect prediction_2_rect = rectFromBoundingBox(predictions[j]);
+            float prediction_2_area = area(prediction_2_rect);
+
+            float overlap = overlappingArea(prediction_1_rect, prediction_2_rect);
+            float smallest_area = min(prediction_1_area, prediction_2_area);
+
+            if (overlap > (overlap_threshold * smallest_area))
+            {
+                passed = false;
+                break;
+            }
+        }
+
+        if (passed)
+        {
+            passed_predictions.push_back(predictions[i]);
+        }
+    }
+
+    for(JsonVariant prediction : passed_predictions)
+    {
+        String boundingBox = prediction["boundingBox"].as<String>();
+        String tag = prediction["tagName"].as<String>();
+        float probability = prediction["probability"].as<float>();
+
+        char buff[32];
+        sprintf(buff, "%s:\t%.2f%%\t%s", tag.c_str(), probability * 100.0, boundingBox.c_str());
+        Serial.println(buff);
+    }
+
+    Serial.print("Counted ");
+    Serial.print(passed_predictions.size());
+    Serial.println(" stock items.");
+}
 
 void detectStock(byte *buffer, uint32_t length)
 {
@@ -78,17 +171,18 @@ void detectStock(byte *buffer, uint32_t length)
         JsonObject obj = doc.as<JsonObject>();
         JsonArray predictions = obj["predictions"].as<JsonArray>();
 
+        std::vector<JsonVariant> passed_predictions;
+
         for(JsonVariant prediction : predictions) 
         {
             float probability = prediction["probability"].as<float>();
             if (probability > threshold)
             {
-                String tag = prediction["tagName"].as<String>();
+                passed_predictions.push_back(prediction);
-                char buff[32];
-                sprintf(buff, "%s:\t%.2f%%", tag.c_str(), probability * 100.0);
-                Serial.println(buff);
             }
         }
+
+        processPredictions(passed_predictions);
     }
 
     httpClient.end();

5-retail/lessons/2-check-stock-device/code-detect/wio-terminal/fruit-quality-detector/src/main.cpp

@@ -1,10 +1,12 @@
 #include <Arduino.h>
 #include <ArduinoJson.h>
 #include <HTTPClient.h>
+#include <list>
 #include <rpcWiFi.h>
 #include "SD/Seeed_SD.h"
 #include <Seeed_FS.h>
 #include <SPI.h>
+#include <vector>
 #include <WiFiClientSecure.h>
 
 #include "config.h"
@@ -59,6 +61,19 @@ void setup()
 
 const float threshold = 0.3f;
 
+void processPredictions(std::vector<JsonVariant> &predictions)
+{
+    for(JsonVariant prediction : predictions)
+    {
+        String tag = prediction["tagName"].as<String>();
+        float probability = prediction["probability"].as<float>();
+
+        char buff[32];
+        sprintf(buff, "%s:\t%.2f%%", tag.c_str(), probability * 100.0);
+        Serial.println(buff);
+    }
+}
+
 void detectStock(byte *buffer, uint32_t length)
 {
     HTTPClient httpClient;
@@ -78,17 +93,18 @@ void detectStock(byte *buffer, uint32_t length)
         JsonObject obj = doc.as<JsonObject>();
         JsonArray predictions = obj["predictions"].as<JsonArray>();
 
+        std::vector<JsonVariant> passed_predictions;
+
         for(JsonVariant prediction : predictions) 
         {
             float probability = prediction["probability"].as<float>();
             if (probability > threshold)
             {
-                String tag = prediction["tagName"].as<String>();
+                passed_predictions.push_back(prediction);
-                char buff[32];
-                sprintf(buff, "%s:\t%.2f%%", tag.c_str(), probability * 100.0);
-                Serial.println(buff);
             }
         }
+
+        processPredictions(passed_predictions);
     }
 
     httpClient.end();

5-retail/lessons/2-check-stock-device/single-board-computer-count-stock.md

@@ -154,8 +154,10 @@ In the image shown above, the bounding boxes have a small overlap. If this overl
 
     Finally the stock count is printed to the console. This could then be sent to an IoT service to alert if the stock levels are low. All of this code is before the bounding boxes are drawn, so you will see the stock predictions without overlaps on the generated images.
 
+    > 💁 This is very simplistic way to remove overlaps, just removing the first one in an overlapping pair. For production code, you would want to put more logic in here, such as considering the overlaps between multiple objects, or if one bounding box is contained by another.
+
 1. Run the app with the camera pointing at some stock on a shelf. The output will indicate the number of bounding boxes without overlaps that exceed the threshold. Try adjusting the `overlap_threshold` value to see predictions being ignored.
 
-> 💁 You can find this code in the [code-count/pi](code-detect/pi) or [code-count/virtual-device](code-detect/virtual-device) folder.
+> 💁 You can find this code in the [code-count/pi](code-count/pi) or [code-count/virtual-device](code-count/virtual-device) folder.
 
 😀 Your stock counter program was a success!

5-retail/lessons/2-check-stock-device/wio-terminal-count-stock.md

@@ -1,3 +1,167 @@
 # Count stock from your IoT device - Wio Terminal
 
-Coming soon!
+A combination of the predictions and their bounding boxes can be used to count stock in an image.
+
+## Count stock
+
+![4 cans of tomato paste with bounding boxes around each can](../../../images/rpi-stock-with-bounding-boxes.jpg)
+
+In the image shown above, the bounding boxes have a small overlap. If this overlap was much larger, then the bounding boxes may indicate the same object. To count the objects correctly, you need to ignore boxes with a significant overlap.
+
+### Task - count stock ignoring overlap
+
+1. Open your `stock-counter` project if it is not already open.
+
+1. Above the `processPredictions` function, add the following code:
+
+    ```cpp
+    const float overlap_threshold = 0.20f;
+    ```
+
+    This defines the percentage overlap allowed before the bounding boxes are considered to be the same object. 0.20 defines a 20% overlap.
+
+1. Below this, and above the `processPredictions` function, add the following code to calculate the overlap between two rectangles:
+
+    ```cpp
+    struct Point {
+        float x, y;
+    };
+
+    struct Rect {
+        Point topLeft, bottomRight;
+    };
+
+    float area(Rect rect)
+    {
+        return abs(rect.bottomRight.x - rect.topLeft.x) * abs(rect.bottomRight.y - rect.topLeft.y);
+    }
+     
+    float overlappingArea(Rect rect1, Rect rect2)
+    {
+        float left = max(rect1.topLeft.x, rect2.topLeft.x);
+        float right = min(rect1.bottomRight.x, rect2.bottomRight.x);
+        float top = max(rect1.topLeft.y, rect2.topLeft.y);
+        float bottom = min(rect1.bottomRight.y, rect2.bottomRight.y);
+    
+    
+        if ( right > left && bottom > top )
+        {
+            return (right-left)*(bottom-top);
+        }
+        
+        return 0.0f;
+    }
+    ```
+
+    This code defines a `Point` struct to store points on the image, and a `Rect` struct to define a rectangle using a top left and bottom right coordinate. It then defines an `area` function that calculates the area of a rectangle from a top left and bottom right coordinate.
+
+    Next it defines a `overlappingArea` function that calculates the overlapping area of 2 rectangles. If they don't overlap, it returns 0.
+
+1. Below the `overlappingArea` function, declare a function to convert a bounding box to a `Rect`:
+
+    ```cpp
+    Rect rectFromBoundingBox(JsonVariant prediction)
+    {
+        JsonObject bounding_box = prediction["boundingBox"].as<JsonObject>();
+    
+        float left = bounding_box["left"].as<float>();
+        float top = bounding_box["top"].as<float>();
+        float width = bounding_box["width"].as<float>();
+        float height = bounding_box["height"].as<float>();
+    
+        Point topLeft = {left, top};
+        Point bottomRight = {left + width, top + height};
+    
+        return {topLeft, bottomRight};
+    }
+    ```
+
+    This takes a prediction from the object detector, extracts the bounding box and uses the values on the bounding box to define a rectangle. The right side is calculated from the left plus the width. The bottom is calculated as the top plus the height.
+
+1. The predictions need to be compared to each other, and if 2 predictions have an overlap of more that the threshold, one of them needs to be deleted. The overlap threshold is a percentage, so needs to be multiplied by the size of the smallest bounding box to check that the overlap exceeds the given percentage of the bounding box, not the given percentage of the whole image. Start by deleting the content of the `processPredictions` function.
+
+1. Add the following to the empty `processPredictions` function:
+
+    ```cpp
+    std::vector<JsonVariant> passed_predictions;
+
+    for (int i = 0; i < predictions.size(); ++i)
+    {
+        Rect prediction_1_rect = rectFromBoundingBox(predictions[i]);
+        float prediction_1_area = area(prediction_1_rect);
+        bool passed = true;
+
+        for (int j = i + 1; j < predictions.size(); ++j)
+        {
+            Rect prediction_2_rect = rectFromBoundingBox(predictions[j]);
+            float prediction_2_area = area(prediction_2_rect);
+
+            float overlap = overlappingArea(prediction_1_rect, prediction_2_rect);
+            float smallest_area = min(prediction_1_area, prediction_2_area);
+
+            if (overlap > (overlap_threshold * smallest_area))
+            {
+                passed = false;
+                break;
+            }
+        }
+
+        if (passed)
+        {
+            passed_predictions.push_back(predictions[i]);
+        }
+    }
+    ```
+
+    This code declares a vector to store the predictions that don't overlap. It then loops through all the predictions, creating a `Rect` from the bounding box.
+
+    Next this code loops through the remaining predictions, starting at the one after the current prediction. This stops predictions being compared more than once - once 1 and 2 have been compared, there's no need to compare 2 with 1, only with 3, 4, etc.
+
+    For each pair of predictions the overlapping area is calculated. This is then compared to the area of the smallest bounding box - if the overlap exceeds the threshold percentage of the smallest bounding box, the prediction is marked as not passed. If after comparing all the overlap, the prediction passes the checks it is added to the `passed_predictions` collection.
+
+    > 💁 This is very simplistic way to remove overlaps, just removing the first one in an overlapping pair. For production code, you would want to put more logic in here, such as considering the overlaps between multiple objects, or if one bounding box is contained by another.
+
+1. After this, add the following code to send details of the passed predictions to the serial monitor:
+
+    ```cpp
+    for(JsonVariant prediction : passed_predictions)
+    {
+        String boundingBox = prediction["boundingBox"].as<String>();
+        String tag = prediction["tagName"].as<String>();
+        float probability = prediction["probability"].as<float>();
+
+        char buff[32];
+        sprintf(buff, "%s:\t%.2f%%\t%s", tag.c_str(), probability * 100.0, boundingBox.c_str());
+        Serial.println(buff);
+    }
+    ```
+
+    This code loops through the passed predictions and prints their details to the serial monitor.
+
+1. Below this, add code to print the number of counted items to the serial monitor:
+
+    ```cpp
+    Serial.print("Counted ");
+    Serial.print(passed_predictions.size());
+    Serial.println(" stock items.");
+    ```
+
+    This could then be sent to an IoT service to alert if the stock levels are low.
+
+1. Upload and run your code. Point the camera at objects on a shelf and press the C button. Try adjusting the `overlap_threshold` value to see predictions being ignored.
+
+    ```output
+    Connecting to WiFi..
+    Connected!
+    Image captured
+    Image read to buffer with length 17416
+    tomato paste:   35.84%  {"left":0.395631,"top":0.215897,"width":0.180768,"height":0.359364}
+    tomato paste:   35.87%  {"left":0.378554,"top":0.583012,"width":0.14824,"height":0.359382}
+    tomato paste:   34.11%  {"left":0.699024,"top":0.592617,"width":0.124411,"height":0.350456}
+    tomato paste:   35.16%  {"left":0.513006,"top":0.647853,"width":0.187472,"height":0.325817}
+    Counted 4 stock items.
+    ```
+
+> 💁 You can find this code in the [code-count/wio-terminal](code-count/wio-terminal) folder.
+
+😀 Your stock counter program was a success!

5-retail/lessons/2-check-stock-device/wio-terminal-object-detector.md

@@ -24,6 +24,12 @@ The code you used to classify images is very similar to the code to detect objec
 
 ### Task - change the code from a classifier to an image detector
 
+1. Add the following include directive to the top of the `main.cpp` file:
+
+    ```cpp
+    #include <vector>
+    ```
+
 1. Rename the `classifyImage` function to `detectStock`, both the name of the function and the call in the `buttonPressed` function.
 
 1. Above the `detectStock` function, declare a threshold to filter out any detections that have a low probability:
@@ -34,15 +40,16 @@ The code you used to classify images is very similar to the code to detect objec
 
     Unlike an image classifier that only returns one result per tag, the object detector will return multiple results, so any with a low probability need to be filtered out.
 
-1. In the `detectStock` function, replace the contents of the `for` loop that loops through the predictions with the following:
+1. Above the `detectStock` function, declare a function to process the predictions:
 
     ```cpp
-    for(JsonVariant prediction : predictions)
+    void processPredictions(std::vector<JsonVariant> &predictions)
     {
-        float probability = prediction["probability"].as<float>();
+        for(JsonVariant prediction : predictions)
-        if (probability > threshold)
         {
             String tag = prediction["tagName"].as<String>();
+            float probability = prediction["probability"].as<float>();
+    
             char buff[32];
             sprintf(buff, "%s:\t%.2f%%", tag.c_str(), probability * 100.0);
             Serial.println(buff);
@@ -50,7 +57,26 @@ The code you used to classify images is very similar to the code to detect objec
     }
     ```
 
-    > 💁 Apart from the threshold, this code is the same as for the image classifier. One difference is the prediction URL that was called. Another difference is the results will return the location of the object, and this will be covered later in this lesson.
+    This takes a list of predictions and prints them to the serial monitor.
+
+1. In the `detectStock` function, replace the contents of the `for` loop that loops through the predictions with the following:
+
+    ```cpp
+    std::vector<JsonVariant> passed_predictions;
+
+    for(JsonVariant prediction : predictions) 
+    {
+        float probability = prediction["probability"].as<float>();
+        if (probability > threshold)
+        {
+            passed_predictions.push_back(prediction);
+        }
+    }
+
+    processPredictions(passed_predictions);
+    ```
+
+    This loops through the predictions, comparing the probability to the threshold. All predictions that have a probability higher than the threshold are added to a `list` and passed to the `processPredictions` function.
 
 1. Upload and run your code. Point the camera at objects on a shelf and press the C button. You will see the output in the serial monitor:
 

quiz-app/src/assets/translations/en.json

@@ -104,7 +104,7 @@
 			},
 			{
 				"id": 3,
-				"title": "Lesson 2 - Introduction to IoT devices: Pre-Lecture Quiz",
+				"title": "Lesson 2 - A deeper dive into IoT: Pre-Lecture Quiz",
 				"quiz": [
 					{
 						"questionText": "The T in IoT stands for:",
@@ -157,7 +157,7 @@
 			},
 			{
 				"id": 4,
-				"title": "Lesson 2 - Introduction to IoT devices: Post-Lecture Quiz",
+				"title": "Lesson 2 - A deeper dive into IoT: Post-Lecture Quiz",
 				"quiz": [
 					{
 						"questionText": "The three steps in a CPU instruction cycle are:",