Jim Bennett
4 years ago
parent
c651999ba2
commit
e5e5771091
@ -0,0 +1,92 @@
|
|||||||
|
import io
|
||||||
|
import time
|
||||||
|
from picamera import PiCamera
|
||||||
|
|
||||||
|
from azure.cognitiveservices.vision.customvision.prediction import CustomVisionPredictionClient
|
||||||
|
from msrest.authentication import ApiKeyCredentials
|
||||||
|
|
||||||
|
from PIL import Image, ImageDraw, ImageColor
|
||||||
|
from shapely.geometry import Polygon
|
||||||
|
|
||||||
|
camera = PiCamera()
|
||||||
|
camera.resolution = (640, 480)
|
||||||
|
camera.rotation = 0
|
||||||
|
|
||||||
|
time.sleep(2)
|
||||||
|
|
||||||
|
image = io.BytesIO()
|
||||||
|
camera.capture(image, 'jpeg')
|
||||||
|
image.seek(0)
|
||||||
|
|
||||||
|
with open('image.jpg', 'wb') as image_file:
|
||||||
|
image_file.write(image.read())
|
||||||
|
|
||||||
|
prediction_url = '<prediction_url>'
|
||||||
|
prediction_key = '<prediction key>'
|
||||||
|
|
||||||
|
parts = prediction_url.split('/')
|
||||||
|
endpoint = 'https://' + parts[2]
|
||||||
|
project_id = parts[6]
|
||||||
|
iteration_name = parts[9]
|
||||||
|
|
||||||
|
prediction_credentials = ApiKeyCredentials(in_headers={"Prediction-key": prediction_key})
|
||||||
|
predictor = CustomVisionPredictionClient(endpoint, prediction_credentials)
|
||||||
|
|
||||||
|
image.seek(0)
|
||||||
|
results = predictor.detect_image(project_id, iteration_name, image)
|
||||||
|
|
||||||
|
threshold = 0.3
|
||||||
|
|
||||||
|
predictions = list(prediction for prediction in results.predictions if prediction.probability > threshold)
|
||||||
|
|
||||||
|
for prediction in predictions:
|
||||||
|
print(f'{prediction.tag_name}:\t{prediction.probability * 100:.2f}%')
|
||||||
|
|
||||||
|
overlap_threshold = 0.002
|
||||||
|
|
||||||
|
def create_polygon(prediction):
|
||||||
|
scale_left = prediction.bounding_box.left
|
||||||
|
scale_top = prediction.bounding_box.top
|
||||||
|
scale_right = prediction.bounding_box.left + prediction.bounding_box.width
|
||||||
|
scale_bottom = prediction.bounding_box.top + prediction.bounding_box.height
|
||||||
|
|
||||||
|
return Polygon([(scale_left, scale_top), (scale_right, scale_top), (scale_right, scale_bottom), (scale_left, scale_bottom)])
|
||||||
|
|
||||||
|
to_delete = []
|
||||||
|
|
||||||
|
for i in range(0, len(predictions)):
|
||||||
|
polygon_1 = create_polygon(predictions[i])
|
||||||
|
|
||||||
|
for j in range(i+1, len(predictions)):
|
||||||
|
polygon_2 = create_polygon(predictions[j])
|
||||||
|
overlap = polygon_1.intersection(polygon_2).area
|
||||||
|
|
||||||
|
smallest_area = min(polygon_1.area, polygon_2.area)
|
||||||
|
|
||||||
|
if overlap > (overlap_threshold * smallest_area):
|
||||||
|
to_delete.append(predictions[i])
|
||||||
|
break
|
||||||
|
|
||||||
|
for d in to_delete:
|
||||||
|
predictions.remove(d)
|
||||||
|
|
||||||
|
print(f'Counted {len(predictions)} stock items')
|
||||||
|
|
||||||
|
|
||||||
|
with Image.open('image.jpg') as im:
|
||||||
|
draw = ImageDraw.Draw(im)
|
||||||
|
|
||||||
|
for prediction in predictions:
|
||||||
|
scale_left = prediction.bounding_box.left
|
||||||
|
scale_top = prediction.bounding_box.top
|
||||||
|
scale_right = prediction.bounding_box.left + prediction.bounding_box.width
|
||||||
|
scale_bottom = prediction.bounding_box.top + prediction.bounding_box.height
|
||||||
|
|
||||||
|
left = scale_left * im.width
|
||||||
|
top = scale_top * im.height
|
||||||
|
right = scale_right * im.width
|
||||||
|
bottom = scale_bottom * im.height
|
||||||
|
|
||||||
|
draw.rectangle([left, top, right, bottom], outline=ImageColor.getrgb('red'), width=2)
|
||||||
|
|
||||||
|
im.save('image.jpg')
|
||||||
@ -0,0 +1,92 @@
|
|||||||
|
from counterfit_connection import CounterFitConnection
|
||||||
|
CounterFitConnection.init('127.0.0.1', 5000)
|
||||||
|
|
||||||
|
import io
|
||||||
|
from counterfit_shims_picamera import PiCamera
|
||||||
|
|
||||||
|
from azure.cognitiveservices.vision.customvision.prediction import CustomVisionPredictionClient
|
||||||
|
from msrest.authentication import ApiKeyCredentials
|
||||||
|
|
||||||
|
from PIL import Image, ImageDraw, ImageColor
|
||||||
|
from shapely.geometry import Polygon
|
||||||
|
|
||||||
|
camera = PiCamera()
|
||||||
|
camera.resolution = (640, 480)
|
||||||
|
camera.rotation = 0
|
||||||
|
|
||||||
|
image = io.BytesIO()
|
||||||
|
camera.capture(image, 'jpeg')
|
||||||
|
image.seek(0)
|
||||||
|
|
||||||
|
with open('image.jpg', 'wb') as image_file:
|
||||||
|
image_file.write(image.read())
|
||||||
|
|
||||||
|
prediction_url = '<prediction_url>'
|
||||||
|
prediction_key = '<prediction key>'
|
||||||
|
|
||||||
|
parts = prediction_url.split('/')
|
||||||
|
endpoint = 'https://' + parts[2]
|
||||||
|
project_id = parts[6]
|
||||||
|
iteration_name = parts[9]
|
||||||
|
|
||||||
|
prediction_credentials = ApiKeyCredentials(in_headers={"Prediction-key": prediction_key})
|
||||||
|
predictor = CustomVisionPredictionClient(endpoint, prediction_credentials)
|
||||||
|
|
||||||
|
image.seek(0)
|
||||||
|
results = predictor.detect_image(project_id, iteration_name, image)
|
||||||
|
|
||||||
|
threshold = 0.3
|
||||||
|
|
||||||
|
predictions = list(prediction for prediction in results.predictions if prediction.probability > threshold)
|
||||||
|
|
||||||
|
for prediction in predictions:
|
||||||
|
print(f'{prediction.tag_name}:\t{prediction.probability * 100:.2f}%')
|
||||||
|
|
||||||
|
overlap_threshold = 0.002
|
||||||
|
|
||||||
|
def create_polygon(prediction):
|
||||||
|
scale_left = prediction.bounding_box.left
|
||||||
|
scale_top = prediction.bounding_box.top
|
||||||
|
scale_right = prediction.bounding_box.left + prediction.bounding_box.width
|
||||||
|
scale_bottom = prediction.bounding_box.top + prediction.bounding_box.height
|
||||||
|
|
||||||
|
return Polygon([(scale_left, scale_top), (scale_right, scale_top), (scale_right, scale_bottom), (scale_left, scale_bottom)])
|
||||||
|
|
||||||
|
to_delete = []
|
||||||
|
|
||||||
|
for i in range(0, len(predictions)):
|
||||||
|
polygon_1 = create_polygon(predictions[i])
|
||||||
|
|
||||||
|
for j in range(i+1, len(predictions)):
|
||||||
|
polygon_2 = create_polygon(predictions[j])
|
||||||
|
overlap = polygon_1.intersection(polygon_2).area
|
||||||
|
|
||||||
|
smallest_area = min(polygon_1.area, polygon_2.area)
|
||||||
|
|
||||||
|
if overlap > (overlap_threshold * smallest_area):
|
||||||
|
to_delete.append(predictions[i])
|
||||||
|
break
|
||||||
|
|
||||||
|
for d in to_delete:
|
||||||
|
predictions.remove(d)
|
||||||
|
|
||||||
|
print(f'Counted {len(predictions)} stock items')
|
||||||
|
|
||||||
|
|
||||||
|
with Image.open('image.jpg') as im:
|
||||||
|
draw = ImageDraw.Draw(im)
|
||||||
|
|
||||||
|
for prediction in predictions:
|
||||||
|
scale_left = prediction.bounding_box.left
|
||||||
|
scale_top = prediction.bounding_box.top
|
||||||
|
scale_right = prediction.bounding_box.left + prediction.bounding_box.width
|
||||||
|
scale_bottom = prediction.bounding_box.top + prediction.bounding_box.height
|
||||||
|
|
||||||
|
left = scale_left * im.width
|
||||||
|
top = scale_top * im.height
|
||||||
|
right = scale_right * im.width
|
||||||
|
bottom = scale_bottom * im.height
|
||||||
|
|
||||||
|
draw.rectangle([left, top, right, bottom], outline=ImageColor.getrgb('red'), width=2)
|
||||||
|
|
||||||
|
im.save('image.jpg')
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
.pio
|
||||||
|
.vscode/.browse.c_cpp.db*
|
||||||
|
.vscode/c_cpp_properties.json
|
||||||
|
.vscode/launch.json
|
||||||
|
.vscode/ipch
|
||||||
@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
// See http://go.microsoft.com/fwlink/?LinkId=827846
|
||||||
|
// for the documentation about the extensions.json format
|
||||||
|
"recommendations": [
|
||||||
|
"platformio.platformio-ide"
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -0,0 +1,39 @@
|
|||||||
|
|
||||||
|
This directory is intended for project header files.
|
||||||
|
|
||||||
|
A header file is a file containing C declarations and macro definitions
|
||||||
|
to be shared between several project source files. You request the use of a
|
||||||
|
header file in your project source file (C, C++, etc) located in `src` folder
|
||||||
|
by including it, with the C preprocessing directive `#include'.
|
||||||
|
|
||||||
|
```src/main.c
|
||||||
|
|
||||||
|
#include "header.h"
|
||||||
|
|
||||||
|
int main (void)
|
||||||
|
{
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Including a header file produces the same results as copying the header file
|
||||||
|
into each source file that needs it. Such copying would be time-consuming
|
||||||
|
and error-prone. With a header file, the related declarations appear
|
||||||
|
in only one place. If they need to be changed, they can be changed in one
|
||||||
|
place, and programs that include the header file will automatically use the
|
||||||
|
new version when next recompiled. The header file eliminates the labor of
|
||||||
|
finding and changing all the copies as well as the risk that a failure to
|
||||||
|
find one copy will result in inconsistencies within a program.
|
||||||
|
|
||||||
|
In C, the usual convention is to give header files names that end with `.h'.
|
||||||
|
It is most portable to use only letters, digits, dashes, and underscores in
|
||||||
|
header file names, and at most one dot.
|
||||||
|
|
||||||
|
Read more about using header files in official GCC documentation:
|
||||||
|
|
||||||
|
* Include Syntax
|
||||||
|
* Include Operation
|
||||||
|
* Once-Only Headers
|
||||||
|
* Computed Includes
|
||||||
|
|
||||||
|
https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html
|
||||||
@ -0,0 +1,46 @@
|
|||||||
|
|
||||||
|
This directory is intended for project specific (private) libraries.
|
||||||
|
PlatformIO will compile them to static libraries and link into executable file.
|
||||||
|
|
||||||
|
The source code of each library should be placed in a an own separate directory
|
||||||
|
("lib/your_library_name/[here are source files]").
|
||||||
|
|
||||||
|
For example, see a structure of the following two libraries `Foo` and `Bar`:
|
||||||
|
|
||||||
|
|--lib
|
||||||
|
| |
|
||||||
|
| |--Bar
|
||||||
|
| | |--docs
|
||||||
|
| | |--examples
|
||||||
|
| | |--src
|
||||||
|
| | |- Bar.c
|
||||||
|
| | |- Bar.h
|
||||||
|
| | |- library.json (optional, custom build options, etc) https://docs.platformio.org/page/librarymanager/config.html
|
||||||
|
| |
|
||||||
|
| |--Foo
|
||||||
|
| | |- Foo.c
|
||||||
|
| | |- Foo.h
|
||||||
|
| |
|
||||||
|
| |- README --> THIS FILE
|
||||||
|
|
|
||||||
|
|- platformio.ini
|
||||||
|
|--src
|
||||||
|
|- main.c
|
||||||
|
|
||||||
|
and a contents of `src/main.c`:
|
||||||
|
```
|
||||||
|
#include <Foo.h>
|
||||||
|
#include <Bar.h>
|
||||||
|
|
||||||
|
int main (void)
|
||||||
|
{
|
||||||
|
...
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
PlatformIO Library Dependency Finder will find automatically dependent
|
||||||
|
libraries scanning project source files.
|
||||||
|
|
||||||
|
More information about PlatformIO Library Dependency Finder
|
||||||
|
- https://docs.platformio.org/page/librarymanager/ldf.html
|
||||||
@ -0,0 +1,26 @@
|
|||||||
|
; PlatformIO Project Configuration File
|
||||||
|
;
|
||||||
|
; Build options: build flags, source filter
|
||||||
|
; Upload options: custom upload port, speed and extra flags
|
||||||
|
; Library options: dependencies, extra library storages
|
||||||
|
; Advanced options: extra scripting
|
||||||
|
;
|
||||||
|
; Please visit documentation for the other options and examples
|
||||||
|
; https://docs.platformio.org/page/projectconf.html
|
||||||
|
|
||||||
|
[env:seeed_wio_terminal]
|
||||||
|
platform = atmelsam
|
||||||
|
board = seeed_wio_terminal
|
||||||
|
framework = arduino
|
||||||
|
lib_deps =
|
||||||
|
seeed-studio/Seeed Arduino rpcWiFi @ 1.0.5
|
||||||
|
seeed-studio/Seeed Arduino FS @ 2.0.3
|
||||||
|
seeed-studio/Seeed Arduino SFUD @ 2.0.1
|
||||||
|
seeed-studio/Seeed Arduino rpcUnified @ 2.1.3
|
||||||
|
seeed-studio/Seeed_Arduino_mbedtls @ 3.0.1
|
||||||
|
seeed-studio/Seeed Arduino RTC @ 2.0.0
|
||||||
|
bblanchon/ArduinoJson @ 6.17.3
|
||||||
|
build_flags =
|
||||||
|
-w
|
||||||
|
-DARDUCAM_SHIELD_V2
|
||||||
|
-DOV2640_CAM
|
||||||
@ -0,0 +1,160 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <ArduCAM.h>
|
||||||
|
#include <Wire.h>
|
||||||
|
|
||||||
|
class Camera
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
Camera(int format, int image_size) : _arducam(OV2640, PIN_SPI_SS)
|
||||||
|
{
|
||||||
|
_format = format;
|
||||||
|
_image_size = image_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool init()
|
||||||
|
{
|
||||||
|
// Reset the CPLD
|
||||||
|
_arducam.write_reg(0x07, 0x80);
|
||||||
|
delay(100);
|
||||||
|
|
||||||
|
_arducam.write_reg(0x07, 0x00);
|
||||||
|
delay(100);
|
||||||
|
|
||||||
|
// Check if the ArduCAM SPI bus is OK
|
||||||
|
_arducam.write_reg(ARDUCHIP_TEST1, 0x55);
|
||||||
|
if (_arducam.read_reg(ARDUCHIP_TEST1) != 0x55)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Change MCU mode
|
||||||
|
_arducam.set_mode(MCU2LCD_MODE);
|
||||||
|
|
||||||
|
uint8_t vid, pid;
|
||||||
|
|
||||||
|
// Check if the camera module type is OV2640
|
||||||
|
_arducam.wrSensorReg8_8(0xff, 0x01);
|
||||||
|
_arducam.rdSensorReg8_8(OV2640_CHIPID_HIGH, &vid);
|
||||||
|
_arducam.rdSensorReg8_8(OV2640_CHIPID_LOW, &pid);
|
||||||
|
if ((vid != 0x26) && ((pid != 0x41) || (pid != 0x42)))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
_arducam.set_format(_format);
|
||||||
|
_arducam.InitCAM();
|
||||||
|
_arducam.OV2640_set_JPEG_size(_image_size);
|
||||||
|
_arducam.OV2640_set_Light_Mode(Auto);
|
||||||
|
_arducam.OV2640_set_Special_effects(Normal);
|
||||||
|
delay(1000);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void startCapture()
|
||||||
|
{
|
||||||
|
_arducam.flush_fifo();
|
||||||
|
_arducam.clear_fifo_flag();
|
||||||
|
_arducam.start_capture();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool captureReady()
|
||||||
|
{
|
||||||
|
return _arducam.get_bit(ARDUCHIP_TRIG, CAP_DONE_MASK);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool readImageToBuffer(byte **buffer, uint32_t &buffer_length)
|
||||||
|
{
|
||||||
|
if (!captureReady()) return false;
|
||||||
|
|
||||||
|
// Get the image file length
|
||||||
|
uint32_t length = _arducam.read_fifo_length();
|
||||||
|
buffer_length = length;
|
||||||
|
|
||||||
|
if (length >= MAX_FIFO_SIZE)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (length == 0)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// create the buffer
|
||||||
|
byte *buf = new byte[length];
|
||||||
|
|
||||||
|
uint8_t temp = 0, temp_last = 0;
|
||||||
|
int i = 0;
|
||||||
|
uint32_t buffer_pos = 0;
|
||||||
|
bool is_header = false;
|
||||||
|
|
||||||
|
_arducam.CS_LOW();
|
||||||
|
_arducam.set_fifo_burst();
|
||||||
|
|
||||||
|
while (length--)
|
||||||
|
{
|
||||||
|
temp_last = temp;
|
||||||
|
temp = SPI.transfer(0x00);
|
||||||
|
//Read JPEG data from FIFO
|
||||||
|
if ((temp == 0xD9) && (temp_last == 0xFF)) //If find the end ,break while,
|
||||||
|
{
|
||||||
|
buf[buffer_pos] = temp;
|
||||||
|
|
||||||
|
buffer_pos++;
|
||||||
|
i++;
|
||||||
|
|
||||||
|
_arducam.CS_HIGH();
|
||||||
|
}
|
||||||
|
if (is_header == true)
|
||||||
|
{
|
||||||
|
//Write image data to buffer if not full
|
||||||
|
if (i < 256)
|
||||||
|
{
|
||||||
|
buf[buffer_pos] = temp;
|
||||||
|
buffer_pos++;
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_arducam.CS_HIGH();
|
||||||
|
|
||||||
|
i = 0;
|
||||||
|
buf[buffer_pos] = temp;
|
||||||
|
|
||||||
|
buffer_pos++;
|
||||||
|
i++;
|
||||||
|
|
||||||
|
_arducam.CS_LOW();
|
||||||
|
_arducam.set_fifo_burst();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if ((temp == 0xD8) & (temp_last == 0xFF))
|
||||||
|
{
|
||||||
|
is_header = true;
|
||||||
|
|
||||||
|
buf[buffer_pos] = temp_last;
|
||||||
|
buffer_pos++;
|
||||||
|
i++;
|
||||||
|
|
||||||
|
buf[buffer_pos] = temp;
|
||||||
|
buffer_pos++;
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_arducam.clear_fifo_flag();
|
||||||
|
|
||||||
|
_arducam.set_format(_format);
|
||||||
|
_arducam.InitCAM();
|
||||||
|
_arducam.OV2640_set_JPEG_size(_image_size);
|
||||||
|
|
||||||
|
// return the buffer
|
||||||
|
*buffer = buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
ArduCAM _arducam;
|
||||||
|
int _format;
|
||||||
|
int _image_size;
|
||||||
|
};
|
||||||
@ -0,0 +1,49 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
using namespace std;
|
||||||
|
|
||||||
|
// WiFi credentials
|
||||||
|
const char *SSID = "<SSID>";
|
||||||
|
const char *PASSWORD = "<PASSWORD>";
|
||||||
|
|
||||||
|
const char *PREDICTION_URL = "<PREDICTION_URL>";
|
||||||
|
const char *PREDICTION_KEY = "<PREDICTION_KEY>";
|
||||||
|
|
||||||
|
// Microsoft Azure DigiCert Global Root G2 global certificate
|
||||||
|
const char *CERTIFICATE =
|
||||||
|
"-----BEGIN CERTIFICATE-----\r\n"
|
||||||
|
"MIIF8zCCBNugAwIBAgIQAueRcfuAIek/4tmDg0xQwDANBgkqhkiG9w0BAQwFADBh\r\n"
|
||||||
|
"MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3\r\n"
|
||||||
|
"d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH\r\n"
|
||||||
|
"MjAeFw0yMDA3MjkxMjMwMDBaFw0yNDA2MjcyMzU5NTlaMFkxCzAJBgNVBAYTAlVT\r\n"
|
||||||
|
"MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKjAoBgNVBAMTIU1pY3Jv\r\n"
|
||||||
|
"c29mdCBBenVyZSBUTFMgSXNzdWluZyBDQSAwNjCCAiIwDQYJKoZIhvcNAQEBBQAD\r\n"
|
||||||
|
"ggIPADCCAgoCggIBALVGARl56bx3KBUSGuPc4H5uoNFkFH4e7pvTCxRi4j/+z+Xb\r\n"
|
||||||
|
"wjEz+5CipDOqjx9/jWjskL5dk7PaQkzItidsAAnDCW1leZBOIi68Lff1bjTeZgMY\r\n"
|
||||||
|
"iwdRd3Y39b/lcGpiuP2d23W95YHkMMT8IlWosYIX0f4kYb62rphyfnAjYb/4Od99\r\n"
|
||||||
|
"ThnhlAxGtfvSbXcBVIKCYfZgqRvV+5lReUnd1aNjRYVzPOoifgSx2fRyy1+pO1Uz\r\n"
|
||||||
|
"aMMNnIOE71bVYW0A1hr19w7kOb0KkJXoALTDDj1ukUEDqQuBfBxReL5mXiu1O7WG\r\n"
|
||||||
|
"0vltg0VZ/SZzctBsdBlx1BkmWYBW261KZgBivrql5ELTKKd8qgtHcLQA5fl6JB0Q\r\n"
|
||||||
|
"gs5XDaWehN86Gps5JW8ArjGtjcWAIP+X8CQaWfaCnuRm6Bk/03PQWhgdi84qwA0s\r\n"
|
||||||
|
"sRfFJwHUPTNSnE8EiGVk2frt0u8PG1pwSQsFuNJfcYIHEv1vOzP7uEOuDydsmCjh\r\n"
|
||||||
|
"lxuoK2n5/2aVR3BMTu+p4+gl8alXoBycyLmj3J/PUgqD8SL5fTCUegGsdia/Sa60\r\n"
|
||||||
|
"N2oV7vQ17wjMN+LXa2rjj/b4ZlZgXVojDmAjDwIRdDUujQu0RVsJqFLMzSIHpp2C\r\n"
|
||||||
|
"Zp7mIoLrySay2YYBu7SiNwL95X6He2kS8eefBBHjzwW/9FxGqry57i71c2cDAgMB\r\n"
|
||||||
|
"AAGjggGtMIIBqTAdBgNVHQ4EFgQU1cFnOsKjnfR3UltZEjgp5lVou6UwHwYDVR0j\r\n"
|
||||||
|
"BBgwFoAUTiJUIBiV5uNu5g/6+rkS7QYXjzkwDgYDVR0PAQH/BAQDAgGGMB0GA1Ud\r\n"
|
||||||
|
"JQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjASBgNVHRMBAf8ECDAGAQH/AgEAMHYG\r\n"
|
||||||
|
"CCsGAQUFBwEBBGowaDAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQu\r\n"
|
||||||
|
"Y29tMEAGCCsGAQUFBzAChjRodHRwOi8vY2FjZXJ0cy5kaWdpY2VydC5jb20vRGln\r\n"
|
||||||
|
"aUNlcnRHbG9iYWxSb290RzIuY3J0MHsGA1UdHwR0MHIwN6A1oDOGMWh0dHA6Ly9j\r\n"
|
||||||
|
"cmwzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydEdsb2JhbFJvb3RHMi5jcmwwN6A1oDOG\r\n"
|
||||||
|
"MWh0dHA6Ly9jcmw0LmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydEdsb2JhbFJvb3RHMi5j\r\n"
|
||||||
|
"cmwwHQYDVR0gBBYwFDAIBgZngQwBAgEwCAYGZ4EMAQICMBAGCSsGAQQBgjcVAQQD\r\n"
|
||||||
|
"AgEAMA0GCSqGSIb3DQEBDAUAA4IBAQB2oWc93fB8esci/8esixj++N22meiGDjgF\r\n"
|
||||||
|
"+rA2LUK5IOQOgcUSTGKSqF9lYfAxPjrqPjDCUPHCURv+26ad5P/BYtXtbmtxJWu+\r\n"
|
||||||
|
"cS5BhMDPPeG3oPZwXRHBJFAkY4O4AF7RIAAUW6EzDflUoDHKv83zOiPfYGcpHc9s\r\n"
|
||||||
|
"kxAInCedk7QSgXvMARjjOqdakor21DTmNIUotxo8kHv5hwRlGhBJwps6fEVi1Bt0\r\n"
|
||||||
|
"trpM/3wYxlr473WSPUFZPgP1j519kLpWOJ8z09wxay+Br29irPcBYv0GMXlHqThy\r\n"
|
||||||
|
"8y4m/HyTQeI2IMvMrQnwqPpY+rLIXyviI2vLoI+4xKE4Rn38ZZ8m\r\n"
|
||||||
|
"-----END CERTIFICATE-----\r\n";
|
||||||
@ -0,0 +1,129 @@
|
|||||||
|
#include <Arduino.h>
|
||||||
|
#include <ArduinoJson.h>
|
||||||
|
#include <HTTPClient.h>
|
||||||
|
#include <rpcWiFi.h>
|
||||||
|
#include "SD/Seeed_SD.h"
|
||||||
|
#include <Seeed_FS.h>
|
||||||
|
#include <SPI.h>
|
||||||
|
#include <WiFiClientSecure.h>
|
||||||
|
|
||||||
|
#include "config.h"
|
||||||
|
#include "camera.h"
|
||||||
|
|
||||||
|
Camera camera = Camera(JPEG, OV2640_640x480);
|
||||||
|
|
||||||
|
WiFiClientSecure client;
|
||||||
|
|
||||||
|
void setupCamera()
|
||||||
|
{
|
||||||
|
pinMode(PIN_SPI_SS, OUTPUT);
|
||||||
|
digitalWrite(PIN_SPI_SS, HIGH);
|
||||||
|
|
||||||
|
Wire.begin();
|
||||||
|
SPI.begin();
|
||||||
|
|
||||||
|
if (!camera.init())
|
||||||
|
{
|
||||||
|
Serial.println("Error setting up the camera!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void connectWiFi()
|
||||||
|
{
|
||||||
|
while (WiFi.status() != WL_CONNECTED)
|
||||||
|
{
|
||||||
|
Serial.println("Connecting to WiFi..");
|
||||||
|
WiFi.begin(SSID, PASSWORD);
|
||||||
|
delay(500);
|
||||||
|
}
|
||||||
|
|
||||||
|
client.setCACert(CERTIFICATE);
|
||||||
|
Serial.println("Connected!");
|
||||||
|
}
|
||||||
|
|
||||||
|
void setup()
|
||||||
|
{
|
||||||
|
Serial.begin(9600);
|
||||||
|
|
||||||
|
while (!Serial)
|
||||||
|
; // Wait for Serial to be ready
|
||||||
|
|
||||||
|
delay(1000);
|
||||||
|
|
||||||
|
connectWiFi();
|
||||||
|
|
||||||
|
setupCamera();
|
||||||
|
|
||||||
|
pinMode(WIO_KEY_C, INPUT_PULLUP);
|
||||||
|
}
|
||||||
|
|
||||||
|
const float threshold = 0.3f;
|
||||||
|
|
||||||
|
void detectStock(byte *buffer, uint32_t length)
|
||||||
|
{
|
||||||
|
HTTPClient httpClient;
|
||||||
|
httpClient.begin(client, PREDICTION_URL);
|
||||||
|
httpClient.addHeader("Content-Type", "application/octet-stream");
|
||||||
|
httpClient.addHeader("Prediction-Key", PREDICTION_KEY);
|
||||||
|
|
||||||
|
int httpResponseCode = httpClient.POST(buffer, length);
|
||||||
|
|
||||||
|
if (httpResponseCode == 200)
|
||||||
|
{
|
||||||
|
String result = httpClient.getString();
|
||||||
|
|
||||||
|
DynamicJsonDocument doc(1024);
|
||||||
|
deserializeJson(doc, result.c_str());
|
||||||
|
|
||||||
|
JsonObject obj = doc.as<JsonObject>();
|
||||||
|
JsonArray predictions = obj["predictions"].as<JsonArray>();
|
||||||
|
|
||||||
|
for(JsonVariant prediction : predictions)
|
||||||
|
{
|
||||||
|
float probability = prediction["probability"].as<float>();
|
||||||
|
if (probability > threshold)
|
||||||
|
{
|
||||||
|
String tag = prediction["tagName"].as<String>();
|
||||||
|
char buff[32];
|
||||||
|
sprintf(buff, "%s:\t%.2f%%", tag.c_str(), probability * 100.0);
|
||||||
|
Serial.println(buff);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
httpClient.end();
|
||||||
|
}
|
||||||
|
|
||||||
|
void buttonPressed()
|
||||||
|
{
|
||||||
|
camera.startCapture();
|
||||||
|
|
||||||
|
while (!camera.captureReady())
|
||||||
|
delay(100);
|
||||||
|
|
||||||
|
Serial.println("Image captured");
|
||||||
|
|
||||||
|
byte *buffer;
|
||||||
|
uint32_t length;
|
||||||
|
|
||||||
|
if (camera.readImageToBuffer(&buffer, length))
|
||||||
|
{
|
||||||
|
Serial.print("Image read to buffer with length ");
|
||||||
|
Serial.println(length);
|
||||||
|
|
||||||
|
detectStock(buffer, length);
|
||||||
|
|
||||||
|
delete (buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void loop()
|
||||||
|
{
|
||||||
|
if (digitalRead(WIO_KEY_C) == LOW)
|
||||||
|
{
|
||||||
|
buttonPressed();
|
||||||
|
delay(2000);
|
||||||
|
}
|
||||||
|
|
||||||
|
delay(200);
|
||||||
|
}
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
|
||||||
|
This directory is intended for PlatformIO Unit Testing and project tests.
|
||||||
|
|
||||||
|
Unit Testing is a software testing method by which individual units of
|
||||||
|
source code, sets of one or more MCU program modules together with associated
|
||||||
|
control data, usage procedures, and operating procedures, are tested to
|
||||||
|
determine whether they are fit for use. Unit testing finds problems early
|
||||||
|
in the development cycle.
|
||||||
|
|
||||||
|
More information about PlatformIO Unit Testing:
|
||||||
|
- https://docs.platformio.org/page/plus/unit-testing.html
|
||||||
@ -0,0 +1,161 @@
|
|||||||
|
# Count stock from your IoT device - Virtual IoT Hardware and Raspberry Pi
|
||||||
|
|
||||||
|
A combination of the predictions and their bounding boxes can be used to count stock in an image
|
||||||
|
|
||||||
|
## Show bounding boxes
|
||||||
|
|
||||||
|
As a helpful debugging step you can not only print out the bounding boxes, but you can also draw them on the image that was written to disk when an image was captured.
|
||||||
|
|
||||||
|
### Task - print the bounding boxes
|
||||||
|
|
||||||
|
1. Ensure the `stock-counter` project is open in VS Code, and the virtual environment is activated if you are using a virtual IoT device.
|
||||||
|
|
||||||
|
1. Change the `print` statement in the `for` loop to the following to print the bounding boxes to the console:
|
||||||
|
|
||||||
|
```python
|
||||||
|
print(f'{prediction.tag_name}:\t{prediction.probability * 100:.2f}%\t{prediction.bounding_box}')
|
||||||
|
```
|
||||||
|
|
||||||
|
1. Run the app with the camera pointing at some stock on a shelf. The bounding boxes will be printed to the console, with left, top, width and height values from 0-1.
|
||||||
|
|
||||||
|
```output
|
||||||
|
pi@raspberrypi:~/stock-counter $ python3 app.py
|
||||||
|
tomato paste: 33.42% {'additional_properties': {}, 'left': 0.3455171, 'top': 0.09916268, 'width': 0.14175442, 'height': 0.29405564}
|
||||||
|
tomato paste: 34.41% {'additional_properties': {}, 'left': 0.48283678, 'top': 0.10242918, 'width': 0.11782813, 'height': 0.27467814}
|
||||||
|
tomato paste: 31.25% {'additional_properties': {}, 'left': 0.4923783, 'top': 0.35007596, 'width': 0.13668466, 'height': 0.28304994}
|
||||||
|
tomato paste: 31.05% {'additional_properties': {}, 'left': 0.36416405, 'top': 0.37494493, 'width': 0.14024884, 'height': 0.26880276}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Task - draw bounding boxes on the image
|
||||||
|
|
||||||
|
1. The Pip package [Pillow](https://pypi.org/project/Pillow/) can be used to draw on images. Install this with the following command:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
pip3 install pillow
|
||||||
|
```
|
||||||
|
|
||||||
|
If you are using a virtual IoT device, make sure to run this from inside the activated virtual environment.
|
||||||
|
|
||||||
|
1. Add the following import statement to the top of the `app.py` file:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from PIL import Image, ImageDraw, ImageColor
|
||||||
|
```
|
||||||
|
|
||||||
|
This imports code needed to edit the image.
|
||||||
|
|
||||||
|
1. Add the following code to the end of the `app.py` file:
|
||||||
|
|
||||||
|
```python
|
||||||
|
with Image.open('image.jpg') as im:
|
||||||
|
draw = ImageDraw.Draw(im)
|
||||||
|
|
||||||
|
for prediction in predictions:
|
||||||
|
scale_left = prediction.bounding_box.left
|
||||||
|
scale_top = prediction.bounding_box.top
|
||||||
|
scale_right = prediction.bounding_box.left + prediction.bounding_box.width
|
||||||
|
scale_bottom = prediction.bounding_box.top + prediction.bounding_box.height
|
||||||
|
|
||||||
|
left = scale_left * im.width
|
||||||
|
top = scale_top * im.height
|
||||||
|
right = scale_right * im.width
|
||||||
|
bottom = scale_bottom * im.height
|
||||||
|
|
||||||
|
draw.rectangle([left, top, right, bottom], outline=ImageColor.getrgb('red'), width=2)
|
||||||
|
|
||||||
|
im.save('image.jpg')
|
||||||
|
```
|
||||||
|
|
||||||
|
This code opens the image that was saved earlier for editing. It then loops through the predictions getting the bounding boxes, and calculates the bottom right coordinate using the bounding box values from 0-1. These are then converted to image coordinates by multiplying by the relevant dimension of the image. For example, if the left value was 0.5 on an image that was 600 pixels wide, this would convert it to 300 (0.5 x 600 = 300).
|
||||||
|
|
||||||
|
Each bounding box is drawn on the image using a red line. Finally the edited image is saved, overwriting the original image.
|
||||||
|
|
||||||
|
1. Run the app with the camera pointing at some stock on a shelf. You will see the `image.jpg` file in the VS Code explorer, and you will be able to select it to see the bounding boxes.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Count stock
|
||||||
|
|
||||||
|
In the image shown above, the bounding boxes have a small overlap. If this overlap was much larger, then the bounding boxes may indicate the same object. To count the objects correctly, you need to ignore boxes with a significant overlap.
|
||||||
|
|
||||||
|
### Task - count stock ignoring overlap
|
||||||
|
|
||||||
|
1. The Pip package [Shapely](https://pypi.org/project/Shapely/) can be used to calculate the intersection. If you are using a Raspberry Pi, you will need to instal a library dependency first:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sudo apt install libgeos-dev
|
||||||
|
```
|
||||||
|
|
||||||
|
1. Install the Shapely Pip package:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
pip3 install shapely
|
||||||
|
```
|
||||||
|
|
||||||
|
If you are using a virtual IoT device, make sure to run this from inside the activated virtual environment.
|
||||||
|
|
||||||
|
1. Add the following import statement to the top of the `app.py` file:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from shapely.geometry import Polygon
|
||||||
|
```
|
||||||
|
|
||||||
|
This imports code needed to create polygons to calculate overlap.
|
||||||
|
|
||||||
|
1. Above the code that draws the bounding boxes, add the following code:
|
||||||
|
|
||||||
|
```python
|
||||||
|
overlap_threshold = 0.20
|
||||||
|
```
|
||||||
|
|
||||||
|
This defines the percentage overlap allowed before the bounding boxes are considered to be the same object. 0.20 defines a 20% overlap.
|
||||||
|
|
||||||
|
1. To calculate overlap using Shapely, the bounding boxes need to be converted into Shapely polygons. Add the following function to do this:
|
||||||
|
|
||||||
|
```python
|
||||||
|
def create_polygon(prediction):
|
||||||
|
scale_left = prediction.bounding_box.left
|
||||||
|
scale_top = prediction.bounding_box.top
|
||||||
|
scale_right = prediction.bounding_box.left + prediction.bounding_box.width
|
||||||
|
scale_bottom = prediction.bounding_box.top + prediction.bounding_box.height
|
||||||
|
|
||||||
|
return Polygon([(scale_left, scale_top), (scale_right, scale_top), (scale_right, scale_bottom), (scale_left, scale_bottom)])
|
||||||
|
```
|
||||||
|
|
||||||
|
This creates a polygon using the bounding box of a prediction.
|
||||||
|
|
||||||
|
1. The logic for removing overlapping objects involves comparing all bounding boxes and if any pairs of predictions have bounding boxes that overlap more than the threshold, delete one of the predictions. To compare all the predictions, you compare prediction 1 with 2, 3, 4, etc., then 2 with 3, 4, etc. The following code does this:
|
||||||
|
|
||||||
|
```python
|
||||||
|
to_delete = []
|
||||||
|
|
||||||
|
for i in range(0, len(predictions)):
|
||||||
|
polygon_1 = create_polygon(predictions[i])
|
||||||
|
|
||||||
|
for j in range(i+1, len(predictions)):
|
||||||
|
polygon_2 = create_polygon(predictions[j])
|
||||||
|
overlap = polygon_1.intersection(polygon_2).area
|
||||||
|
|
||||||
|
smallest_area = min(polygon_1.area, polygon_2.area)
|
||||||
|
|
||||||
|
if overlap > (overlap_threshold * smallest_area):
|
||||||
|
to_delete.append(predictions[i])
|
||||||
|
break
|
||||||
|
|
||||||
|
for d in to_delete:
|
||||||
|
predictions.remove(d)
|
||||||
|
|
||||||
|
print(f'Counted {len(predictions)} stock items')
|
||||||
|
```
|
||||||
|
|
||||||
|
The overlap is calculated using the Shapely `Polygon.intersection` method that returns a polygon that has the overlap. The area is then calculated from this polygon. This overlap threshold is not an absolute value, but needs to be a percentage of the bounding box, so the smallest bounding box is found, and the overlap threshold is used to calculate what area the overlap can be to not exceed the percentage overlap threshold of the smallest bounding box. If the overlap exceeds this, the prediction is marked for deletion.
|
||||||
|
|
||||||
|
Once a prediction has been marked for deletion it doesn't need to be checked again, so the inner loop breaks out to check the next prediction. You can't delete items from a list whilst iterating through it, so the bounding boxes that overlap more than the threshold are added to the `to_delete` list, then deleted at the end.
|
||||||
|
|
||||||
|
Finally the stock count is printed to the console. This could then be sent to an IoT service to alert if the stock levels are low. All of this code is before the bounding boxes are drawn, so you will see the stock predictions without overlaps on the generated images.
|
||||||
|
|
||||||
|
1. Run the app with the camera pointing at some stock on a shelf. The output will indicate the number of bounding boxes without overlaps that exceed the threshold. Try adjusting the `overlap_threshold` value to see predictions being ignored.
|
||||||
|
|
||||||
|
> 💁 You can find this code in the [code-count/pi](code-detect/pi) or [code-count/virtual-device](code-detect/virtual-device) folder.
|
||||||
|
|
||||||
|
😀 Your stock counter program was a success!
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
# Count stock from your IoT device - Wio Terminal
|
||||||
|
|
||||||
|
Coming soon!
|
||||||
|
After Width: | Height: | Size: 31 KiB |
Loading…
Reference in new issue