From 7964de281e138c980dbca0afe95bc7c89d7a7552 Mon Sep 17 00:00:00 2001 From: Jim Bennett Date: Thu, 1 Jul 2021 10:28:35 -0700 Subject: [PATCH] Finishing stock counting --- .../lessons/2-check-stock-device/README.md | 2 +- .../fruit-quality-detector/src/main.cpp | 106 ++++++++++- .../fruit-quality-detector/src/main.cpp | 24 ++- .../single-board-computer-count-stock.md | 4 +- .../wio-terminal-count-stock.md | 166 +++++++++++++++++- .../wio-terminal-object-detector.md | 36 +++- README.md | 2 +- quiz-app/src/assets/translations/en.json | 4 +- 8 files changed, 323 insertions(+), 21 deletions(-) diff --git a/5-retail/lessons/2-check-stock-device/README.md b/5-retail/lessons/2-check-stock-device/README.md index a7883c0f..0a5d37d1 100644 --- a/5-retail/lessons/2-check-stock-device/README.md +++ b/5-retail/lessons/2-check-stock-device/README.md @@ -97,7 +97,7 @@ In the image above, 4 cans of tomato paste were detected. In the results a red s ✅ Open the predictions in Custom Vision and check out the bounding boxes. -Bounding boxes are defined with 4 values - top, left, height and width. These values are on a scale of 0-1, representing the positions as a percentage of the size of the image. +Bounding boxes are defined with 4 values - top, left, height and width. These values are on a scale of 0-1, representing the positions as a percentage of the size of the image. The origin (the 0,0 position) is the top left of the image, so the top value is the distance from the top, and the bottom of the bounding box is the top plus the height. ![A bounding box around a can of tomato paste](../../../images/bounding-box.png) diff --git a/5-retail/lessons/2-check-stock-device/code-count/wio-terminal/fruit-quality-detector/src/main.cpp b/5-retail/lessons/2-check-stock-device/code-count/wio-terminal/fruit-quality-detector/src/main.cpp index 5c3951a1..b8bf581f 100644 --- a/5-retail/lessons/2-check-stock-device/code-count/wio-terminal/fruit-quality-detector/src/main.cpp +++ b/5-retail/lessons/2-check-stock-device/code-count/wio-terminal/fruit-quality-detector/src/main.cpp @@ -5,6 +5,7 @@ #include "SD/Seeed_SD.h" #include #include +#include #include #include "config.h" @@ -57,7 +58,99 @@ void setup() pinMode(WIO_KEY_C, INPUT_PULLUP); } -const float threshold = 0.3f; +const float threshold = 0.0f; +const float overlap_threshold = 0.20f; + +struct Point { + float x, y; +}; + +struct Rect { + Point topLeft, bottomRight; +}; + +float area(Rect rect) +{ + return abs(rect.bottomRight.x - rect.topLeft.x) * abs(rect.bottomRight.y - rect.topLeft.y); +} + +float overlappingArea(Rect rect1, Rect rect2) +{ + float left = max(rect1.topLeft.x, rect2.topLeft.x); + float right = min(rect1.bottomRight.x, rect2.bottomRight.x); + float top = max(rect1.topLeft.y, rect2.topLeft.y); + float bottom = min(rect1.bottomRight.y, rect2.bottomRight.y); + + + if ( right > left && bottom > top ) + { + return (right-left)*(bottom-top); + } + + return 0.0f; +} + +Rect rectFromBoundingBox(JsonVariant prediction) +{ + JsonObject bounding_box = prediction["boundingBox"].as(); + + float left = bounding_box["left"].as(); + float top = bounding_box["top"].as(); + float width = bounding_box["width"].as(); + float height = bounding_box["height"].as(); + + Point topLeft = {left, top}; + Point bottomRight = {left + width, top + height}; + + return {topLeft, bottomRight}; +} + +void processPredictions(std::vector &predictions) +{ + std::vector passed_predictions; + + for (int i = 0; i < predictions.size(); ++i) + { + Rect prediction_1_rect = rectFromBoundingBox(predictions[i]); + float prediction_1_area = area(prediction_1_rect); + bool passed = true; + + for (int j = i + 1; j < predictions.size(); ++j) + { + Rect prediction_2_rect = rectFromBoundingBox(predictions[j]); + float prediction_2_area = area(prediction_2_rect); + + float overlap = overlappingArea(prediction_1_rect, prediction_2_rect); + float smallest_area = min(prediction_1_area, prediction_2_area); + + if (overlap > (overlap_threshold * smallest_area)) + { + passed = false; + break; + } + } + + if (passed) + { + passed_predictions.push_back(predictions[i]); + } + } + + for(JsonVariant prediction : passed_predictions) + { + String boundingBox = prediction["boundingBox"].as(); + String tag = prediction["tagName"].as(); + float probability = prediction["probability"].as(); + + char buff[32]; + sprintf(buff, "%s:\t%.2f%%\t%s", tag.c_str(), probability * 100.0, boundingBox.c_str()); + Serial.println(buff); + } + + Serial.print("Counted "); + Serial.print(passed_predictions.size()); + Serial.println(" stock items."); +} void detectStock(byte *buffer, uint32_t length) { @@ -78,17 +171,18 @@ void detectStock(byte *buffer, uint32_t length) JsonObject obj = doc.as(); JsonArray predictions = obj["predictions"].as(); + std::vector passed_predictions; + for(JsonVariant prediction : predictions) { float probability = prediction["probability"].as(); if (probability > threshold) { - String tag = prediction["tagName"].as(); - char buff[32]; - sprintf(buff, "%s:\t%.2f%%", tag.c_str(), probability * 100.0); - Serial.println(buff); + passed_predictions.push_back(prediction); } } + + processPredictions(passed_predictions); } httpClient.end(); @@ -126,4 +220,4 @@ void loop() } delay(200); -} \ No newline at end of file +} diff --git a/5-retail/lessons/2-check-stock-device/code-detect/wio-terminal/fruit-quality-detector/src/main.cpp b/5-retail/lessons/2-check-stock-device/code-detect/wio-terminal/fruit-quality-detector/src/main.cpp index 5c3951a1..d507be7d 100644 --- a/5-retail/lessons/2-check-stock-device/code-detect/wio-terminal/fruit-quality-detector/src/main.cpp +++ b/5-retail/lessons/2-check-stock-device/code-detect/wio-terminal/fruit-quality-detector/src/main.cpp @@ -1,10 +1,12 @@ #include #include #include +#include #include #include "SD/Seeed_SD.h" #include #include +#include #include #include "config.h" @@ -59,6 +61,19 @@ void setup() const float threshold = 0.3f; +void processPredictions(std::vector &predictions) +{ + for(JsonVariant prediction : predictions) + { + String tag = prediction["tagName"].as(); + float probability = prediction["probability"].as(); + + char buff[32]; + sprintf(buff, "%s:\t%.2f%%", tag.c_str(), probability * 100.0); + Serial.println(buff); + } +} + void detectStock(byte *buffer, uint32_t length) { HTTPClient httpClient; @@ -78,17 +93,18 @@ void detectStock(byte *buffer, uint32_t length) JsonObject obj = doc.as(); JsonArray predictions = obj["predictions"].as(); + std::vector passed_predictions; + for(JsonVariant prediction : predictions) { float probability = prediction["probability"].as(); if (probability > threshold) { - String tag = prediction["tagName"].as(); - char buff[32]; - sprintf(buff, "%s:\t%.2f%%", tag.c_str(), probability * 100.0); - Serial.println(buff); + passed_predictions.push_back(prediction); } } + + processPredictions(passed_predictions); } httpClient.end(); diff --git a/5-retail/lessons/2-check-stock-device/single-board-computer-count-stock.md b/5-retail/lessons/2-check-stock-device/single-board-computer-count-stock.md index a44ee9d4..a6a627da 100644 --- a/5-retail/lessons/2-check-stock-device/single-board-computer-count-stock.md +++ b/5-retail/lessons/2-check-stock-device/single-board-computer-count-stock.md @@ -154,8 +154,10 @@ In the image shown above, the bounding boxes have a small overlap. If this overl Finally the stock count is printed to the console. This could then be sent to an IoT service to alert if the stock levels are low. All of this code is before the bounding boxes are drawn, so you will see the stock predictions without overlaps on the generated images. + > 💁 This is very simplistic way to remove overlaps, just removing the first one in an overlapping pair. For production code, you would want to put more logic in here, such as considering the overlaps between multiple objects, or if one bounding box is contained by another. + 1. Run the app with the camera pointing at some stock on a shelf. The output will indicate the number of bounding boxes without overlaps that exceed the threshold. Try adjusting the `overlap_threshold` value to see predictions being ignored. -> 💁 You can find this code in the [code-count/pi](code-detect/pi) or [code-count/virtual-device](code-detect/virtual-device) folder. +> 💁 You can find this code in the [code-count/pi](code-count/pi) or [code-count/virtual-device](code-count/virtual-device) folder. 😀 Your stock counter program was a success! diff --git a/5-retail/lessons/2-check-stock-device/wio-terminal-count-stock.md b/5-retail/lessons/2-check-stock-device/wio-terminal-count-stock.md index 35e363b0..c6bc9b3c 100644 --- a/5-retail/lessons/2-check-stock-device/wio-terminal-count-stock.md +++ b/5-retail/lessons/2-check-stock-device/wio-terminal-count-stock.md @@ -1,3 +1,167 @@ # Count stock from your IoT device - Wio Terminal -Coming soon! \ No newline at end of file +A combination of the predictions and their bounding boxes can be used to count stock in an image. + +## Count stock + +![4 cans of tomato paste with bounding boxes around each can](../../../images/rpi-stock-with-bounding-boxes.jpg) + +In the image shown above, the bounding boxes have a small overlap. If this overlap was much larger, then the bounding boxes may indicate the same object. To count the objects correctly, you need to ignore boxes with a significant overlap. + +### Task - count stock ignoring overlap + +1. Open your `stock-counter` project if it is not already open. + +1. Above the `processPredictions` function, add the following code: + + ```cpp + const float overlap_threshold = 0.20f; + ``` + + This defines the percentage overlap allowed before the bounding boxes are considered to be the same object. 0.20 defines a 20% overlap. + +1. Below this, and above the `processPredictions` function, add the following code to calculate the overlap between two rectangles: + + ```cpp + struct Point { + float x, y; + }; + + struct Rect { + Point topLeft, bottomRight; + }; + + float area(Rect rect) + { + return abs(rect.bottomRight.x - rect.topLeft.x) * abs(rect.bottomRight.y - rect.topLeft.y); + } + + float overlappingArea(Rect rect1, Rect rect2) + { + float left = max(rect1.topLeft.x, rect2.topLeft.x); + float right = min(rect1.bottomRight.x, rect2.bottomRight.x); + float top = max(rect1.topLeft.y, rect2.topLeft.y); + float bottom = min(rect1.bottomRight.y, rect2.bottomRight.y); + + + if ( right > left && bottom > top ) + { + return (right-left)*(bottom-top); + } + + return 0.0f; + } + ``` + + This code defines a `Point` struct to store points on the image, and a `Rect` struct to define a rectangle using a top left and bottom right coordinate. It then defines an `area` function that calculates the area of a rectangle from a top left and bottom right coordinate. + + Next it defines a `overlappingArea` function that calculates the overlapping area of 2 rectangles. If they don't overlap, it returns 0. + +1. Below the `overlappingArea` function, declare a function to convert a bounding box to a `Rect`: + + ```cpp + Rect rectFromBoundingBox(JsonVariant prediction) + { + JsonObject bounding_box = prediction["boundingBox"].as(); + + float left = bounding_box["left"].as(); + float top = bounding_box["top"].as(); + float width = bounding_box["width"].as(); + float height = bounding_box["height"].as(); + + Point topLeft = {left, top}; + Point bottomRight = {left + width, top + height}; + + return {topLeft, bottomRight}; + } + ``` + + This takes a prediction from the object detector, extracts the bounding box and uses the values on the bounding box to define a rectangle. The right side is calculated from the left plus the width. The bottom is calculated as the top plus the height. + +1. The predictions need to be compared to each other, and if 2 predictions have an overlap of more that the threshold, one of them needs to be deleted. The overlap threshold is a percentage, so needs to be multiplied by the size of the smallest bounding box to check that the overlap exceeds the given percentage of the bounding box, not the given percentage of the whole image. Start by deleting the content of the `processPredictions` function. + +1. Add the following to the empty `processPredictions` function: + + ```cpp + std::vector passed_predictions; + + for (int i = 0; i < predictions.size(); ++i) + { + Rect prediction_1_rect = rectFromBoundingBox(predictions[i]); + float prediction_1_area = area(prediction_1_rect); + bool passed = true; + + for (int j = i + 1; j < predictions.size(); ++j) + { + Rect prediction_2_rect = rectFromBoundingBox(predictions[j]); + float prediction_2_area = area(prediction_2_rect); + + float overlap = overlappingArea(prediction_1_rect, prediction_2_rect); + float smallest_area = min(prediction_1_area, prediction_2_area); + + if (overlap > (overlap_threshold * smallest_area)) + { + passed = false; + break; + } + } + + if (passed) + { + passed_predictions.push_back(predictions[i]); + } + } + ``` + + This code declares a vector to store the predictions that don't overlap. It then loops through all the predictions, creating a `Rect` from the bounding box. + + Next this code loops through the remaining predictions, starting at the one after the current prediction. This stops predictions being compared more than once - once 1 and 2 have been compared, there's no need to compare 2 with 1, only with 3, 4, etc. + + For each pair of predictions the overlapping area is calculated. This is then compared to the area of the smallest bounding box - if the overlap exceeds the threshold percentage of the smallest bounding box, the prediction is marked as not passed. If after comparing all the overlap, the prediction passes the checks it is added to the `passed_predictions` collection. + + > 💁 This is very simplistic way to remove overlaps, just removing the first one in an overlapping pair. For production code, you would want to put more logic in here, such as considering the overlaps between multiple objects, or if one bounding box is contained by another. + +1. After this, add the following code to send details of the passed predictions to the serial monitor: + + ```cpp + for(JsonVariant prediction : passed_predictions) + { + String boundingBox = prediction["boundingBox"].as(); + String tag = prediction["tagName"].as(); + float probability = prediction["probability"].as(); + + char buff[32]; + sprintf(buff, "%s:\t%.2f%%\t%s", tag.c_str(), probability * 100.0, boundingBox.c_str()); + Serial.println(buff); + } + ``` + + This code loops through the passed predictions and prints their details to the serial monitor. + +1. Below this, add code to print the number of counted items to the serial monitor: + + ```cpp + Serial.print("Counted "); + Serial.print(passed_predictions.size()); + Serial.println(" stock items."); + ``` + + This could then be sent to an IoT service to alert if the stock levels are low. + +1. Upload and run your code. Point the camera at objects on a shelf and press the C button. Try adjusting the `overlap_threshold` value to see predictions being ignored. + + ```output + Connecting to WiFi.. + Connected! + Image captured + Image read to buffer with length 17416 + tomato paste: 35.84% {"left":0.395631,"top":0.215897,"width":0.180768,"height":0.359364} + tomato paste: 35.87% {"left":0.378554,"top":0.583012,"width":0.14824,"height":0.359382} + tomato paste: 34.11% {"left":0.699024,"top":0.592617,"width":0.124411,"height":0.350456} + tomato paste: 35.16% {"left":0.513006,"top":0.647853,"width":0.187472,"height":0.325817} + Counted 4 stock items. + ``` + +> 💁 You can find this code in the [code-count/wio-terminal](code-count/wio-terminal) folder. + +😀 Your stock counter program was a success! diff --git a/5-retail/lessons/2-check-stock-device/wio-terminal-object-detector.md b/5-retail/lessons/2-check-stock-device/wio-terminal-object-detector.md index 1e7c3ae7..5c6e7fec 100644 --- a/5-retail/lessons/2-check-stock-device/wio-terminal-object-detector.md +++ b/5-retail/lessons/2-check-stock-device/wio-terminal-object-detector.md @@ -24,6 +24,12 @@ The code you used to classify images is very similar to the code to detect objec ### Task - change the code from a classifier to an image detector +1. Add the following include directive to the top of the `main.cpp` file: + + ```cpp + #include + ``` + 1. Rename the `classifyImage` function to `detectStock`, both the name of the function and the call in the `buttonPressed` function. 1. Above the `detectStock` function, declare a threshold to filter out any detections that have a low probability: @@ -34,15 +40,16 @@ The code you used to classify images is very similar to the code to detect objec Unlike an image classifier that only returns one result per tag, the object detector will return multiple results, so any with a low probability need to be filtered out. -1. In the `detectStock` function, replace the contents of the `for` loop that loops through the predictions with the following: +1. Above the `detectStock` function, declare a function to process the predictions: ```cpp - for(JsonVariant prediction : predictions) + void processPredictions(std::vector &predictions) { - float probability = prediction["probability"].as(); - if (probability > threshold) + for(JsonVariant prediction : predictions) { String tag = prediction["tagName"].as(); + float probability = prediction["probability"].as(); + char buff[32]; sprintf(buff, "%s:\t%.2f%%", tag.c_str(), probability * 100.0); Serial.println(buff); @@ -50,7 +57,26 @@ The code you used to classify images is very similar to the code to detect objec } ``` - > 💁 Apart from the threshold, this code is the same as for the image classifier. One difference is the prediction URL that was called. Another difference is the results will return the location of the object, and this will be covered later in this lesson. + This takes a list of predictions and prints them to the serial monitor. + +1. In the `detectStock` function, replace the contents of the `for` loop that loops through the predictions with the following: + + ```cpp + std::vector passed_predictions; + + for(JsonVariant prediction : predictions) + { + float probability = prediction["probability"].as(); + if (probability > threshold) + { + passed_predictions.push_back(prediction); + } + } + + processPredictions(passed_predictions); + ``` + + This loops through the predictions, comparing the probability to the threshold. All predictions that have a probability higher than the threshold are added to a `list` and passed to the `processPredictions` function. 1. Upload and run your code. Point the camera at objects on a shelf and press the C button. You will see the output in the serial monitor: diff --git a/README.md b/README.md index 163aeb89..2993aeca 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ We have two choices of IoT hardware to use for the projects depending on persona | | Project Name | Concepts Taught | Learning Objectives | Linked Lesson | | :-: | :----------: | :-------------: | ------------------- | :-----------: | | 01 | [Getting started](./1-getting-started) | Introduction to IoT | Learn the basic principles of IoT and the basic building blocks of IoT solutions such as sensors and cloud services whilst you are setting up your first IoT device | [Introduction to IoT](./1-getting-started/lessons/1-introduction-to-iot/README.md) | -| 02 | [Getting started](./1-getting-started) | A deeper dive into IoT| Learn more about the components of an IoT system, as well as microcontrollers and single-board computers | [A deeper dive into IoT](./1-getting-started/lessons/2-deeper-dive/README.md) | +| 02 | [Getting started](./1-getting-started) | A deeper dive into IoT | Learn more about the components of an IoT system, as well as microcontrollers and single-board computers | [A deeper dive into IoT](./1-getting-started/lessons/2-deeper-dive/README.md) | | 03 | [Getting started](./1-getting-started) | Interact with the physical world with sensors and actuators | Learn about sensors to gather data from the physical world, and actuators to send feedback, whilst you build a nightlight | [Interact with the physical world with sensors and actuators](./1-getting-started/lessons/3-sensors-and-actuators/README.md) | | 04 | [Getting started](./1-getting-started) | Connect your device to the Internet | Learn about how to connect an IoT device to the Internet to send and receive messages by connecting your nightlight to an MQTT broker | [Connect your device to the Internet](./1-getting-started/lessons/4-connect-internet/README.md) | | 05 | [Farm](./2-farm) | Predict plant growth | Learn how to predict plant growth using temperature data captured by an IoT device | [Predict plant growth](./2-farm/lessons/1-predict-plant-growth/README.md) | diff --git a/quiz-app/src/assets/translations/en.json b/quiz-app/src/assets/translations/en.json index b7d08ba0..d22b5d2e 100644 --- a/quiz-app/src/assets/translations/en.json +++ b/quiz-app/src/assets/translations/en.json @@ -104,7 +104,7 @@ }, { "id": 3, - "title": "Lesson 2 - Introduction to IoT devices: Pre-Lecture Quiz", + "title": "Lesson 2 - A deeper dive into IoT: Pre-Lecture Quiz", "quiz": [ { "questionText": "The T in IoT stands for:", @@ -157,7 +157,7 @@ }, { "id": 4, - "title": "Lesson 2 - Introduction to IoT devices: Post-Lecture Quiz", + "title": "Lesson 2 - A deeper dive into IoT: Post-Lecture Quiz", "quiz": [ { "questionText": "The three steps in a CPU instruction cycle are:",