diff --git a/1-getting-started/lessons/4-connect-internet/code-commands/wio-terminal/nightlight/src/main.cpp b/1-getting-started/lessons/4-connect-internet/code-commands/wio-terminal/nightlight/src/main.cpp index e8c1ac21..8f0fd6b5 100644 --- a/1-getting-started/lessons/4-connect-internet/code-commands/wio-terminal/nightlight/src/main.cpp +++ b/1-getting-started/lessons/4-connect-internet/code-commands/wio-terminal/nightlight/src/main.cpp @@ -100,8 +100,7 @@ void loop() doc["light"] = light; string telemetry; - JsonObject obj = doc.as(); - serializeJson(obj, telemetry); + serializeJson(doc, telemetry); Serial.print("Sending telemetry "); Serial.println(telemetry.c_str()); diff --git a/1-getting-started/lessons/4-connect-internet/code-telemetry/wio-terminal/nightlight/src/main.cpp b/1-getting-started/lessons/4-connect-internet/code-telemetry/wio-terminal/nightlight/src/main.cpp index 5bf64b3e..81bcfff4 100644 --- a/1-getting-started/lessons/4-connect-internet/code-telemetry/wio-terminal/nightlight/src/main.cpp +++ b/1-getting-started/lessons/4-connect-internet/code-telemetry/wio-terminal/nightlight/src/main.cpp @@ -74,8 +74,7 @@ void loop() doc["light"] = light; string telemetry; - JsonObject obj = doc.as(); - serializeJson(obj, telemetry); + serializeJson(doc, telemetry); Serial.print("Sending telemetry "); Serial.println(telemetry.c_str()); diff --git a/1-getting-started/lessons/4-connect-internet/wio-terminal-telemetry.md b/1-getting-started/lessons/4-connect-internet/wio-terminal-telemetry.md index 12897cf1..86dbbe62 100644 --- a/1-getting-started/lessons/4-connect-internet/wio-terminal-telemetry.md +++ b/1-getting-started/lessons/4-connect-internet/wio-terminal-telemetry.md @@ -53,8 +53,7 @@ Publish telemetry to the MQTT broker. doc["light"] = light; string telemetry; - JsonObject obj = doc.as(); - serializeJson(obj, telemetry); + serializeJson(doc, telemetry); Serial.print("Sending telemetry "); Serial.println(telemetry.c_str()); diff --git a/2-farm/lessons/1-predict-plant-growth/code-publish-temperature/wio-terminal/temperature-sensor/src/main.cpp b/2-farm/lessons/1-predict-plant-growth/code-publish-temperature/wio-terminal/temperature-sensor/src/main.cpp index c76c516b..21192256 100644 --- a/2-farm/lessons/1-predict-plant-growth/code-publish-temperature/wio-terminal/temperature-sensor/src/main.cpp +++ b/2-farm/lessons/1-predict-plant-growth/code-publish-temperature/wio-terminal/temperature-sensor/src/main.cpp @@ -77,8 +77,7 @@ void loop() doc["temperature"] = temp_hum_val[1]; string telemetry; - JsonObject obj = doc.as(); - serializeJson(obj, telemetry); + serializeJson(doc, telemetry); Serial.print("Sending telemetry "); Serial.println(telemetry.c_str()); diff --git a/2-farm/lessons/3-automated-plant-watering/code-mqtt/wio-terminal/soil-moisture-sensor/src/main.cpp b/2-farm/lessons/3-automated-plant-watering/code-mqtt/wio-terminal/soil-moisture-sensor/src/main.cpp index fd4bac4a..486d7db8 100644 --- a/2-farm/lessons/3-automated-plant-watering/code-mqtt/wio-terminal/soil-moisture-sensor/src/main.cpp +++ b/2-farm/lessons/3-automated-plant-watering/code-mqtt/wio-terminal/soil-moisture-sensor/src/main.cpp @@ -100,8 +100,7 @@ void loop() doc["soil_moisture"] = soil_moisture; string telemetry; - JsonObject obj = doc.as(); - serializeJson(obj, telemetry); + serializeJson(doc, telemetry); Serial.print("Sending telemetry "); Serial.println(telemetry.c_str()); diff --git a/2-farm/lessons/4-migrate-your-plant-to-the-cloud/code/wio-terminal/soil-moisture-sensor/src/main.cpp b/2-farm/lessons/4-migrate-your-plant-to-the-cloud/code/wio-terminal/soil-moisture-sensor/src/main.cpp index 23ef9191..7d18409f 100644 --- a/2-farm/lessons/4-migrate-your-plant-to-the-cloud/code/wio-terminal/soil-moisture-sensor/src/main.cpp +++ b/2-farm/lessons/4-migrate-your-plant-to-the-cloud/code/wio-terminal/soil-moisture-sensor/src/main.cpp @@ -120,8 +120,7 @@ void loop() doc["soil_moisture"] = soil_moisture; string telemetry; - JsonObject obj = doc.as(); - serializeJson(obj, telemetry); + serializeJson(doc, telemetry); Serial.print("Sending telemetry "); Serial.println(telemetry.c_str()); diff --git a/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/get-voices/__init__.py b/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/get-voices/__init__.py new file mode 100644 index 00000000..44c81900 --- /dev/null +++ b/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/get-voices/__init__.py @@ -0,0 +1,26 @@ +import json +import os +import requests + +import azure.functions as func + +def main(req: func.HttpRequest) -> func.HttpResponse: + location = os.environ['SPEECH_LOCATION'] + speech_key = os.environ['SPEECH_KEY'] + + req_body = req.get_json() + language = req_body['language'] + + url = f'https://{location}.tts.speech.microsoft.com/cognitiveservices/voices/list' + + headers = { + 'Ocp-Apim-Subscription-Key': speech_key + } + + response = requests.get(url, headers=headers) + voices_json = json.loads(response.text) + + voices = filter(lambda x: x['Locale'].lower() == language.lower(), voices_json) + voices = map(lambda x: x['ShortName'], voices) + + return func.HttpResponse(json.dumps(list(voices)), status_code=200) diff --git a/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/get-voices/function.json b/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/get-voices/function.json new file mode 100644 index 00000000..d9019652 --- /dev/null +++ b/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/get-voices/function.json @@ -0,0 +1,20 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "function", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get", + "post" + ] + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] +} \ No newline at end of file diff --git a/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/local.settings.json b/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/local.settings.json index ee6b34ce..afc05864 100644 --- a/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/local.settings.json +++ b/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/local.settings.json @@ -5,6 +5,8 @@ "AzureWebJobsStorage": "", "LUIS_KEY": "", "LUIS_ENDPOINT_URL": "", - "LUIS_APP_ID": "" + "LUIS_APP_ID": "", + "SPEECH_KEY": "", + "SPEECH_LOCATION": "" } } \ No newline at end of file diff --git a/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/requirements.txt b/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/requirements.txt index d0405a38..a2596be3 100644 --- a/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/requirements.txt +++ b/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/requirements.txt @@ -1,4 +1,5 @@ # Do not include azure-functions-worker as it may conflict with the Azure Functions platform azure-functions -azure-cognitiveservices-language-luis \ No newline at end of file +azure-cognitiveservices-language-luis +librosa \ No newline at end of file diff --git a/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/text-to-speech/__init__.py b/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/text-to-speech/__init__.py new file mode 100644 index 00000000..f09404f3 --- /dev/null +++ b/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/text-to-speech/__init__.py @@ -0,0 +1,52 @@ +import io +import os +import requests + +import librosa +import soundfile as sf +import azure.functions as func + +location = os.environ['SPEECH_LOCATION'] +speech_key = os.environ['SPEECH_KEY'] + +def get_access_token(): + headers = { + 'Ocp-Apim-Subscription-Key': speech_key + } + + token_endpoint = f'https://{location}.api.cognitive.microsoft.com/sts/v1.0/issuetoken' + response = requests.post(token_endpoint, headers=headers) + return str(response.text) + +playback_format = 'riff-48khz-16bit-mono-pcm' + +def main(req: func.HttpRequest) -> func.HttpResponse: + req_body = req.get_json() + language = req_body['language'] + voice = req_body['voice'] + text = req_body['text'] + + url = f'https://{location}.tts.speech.microsoft.com/cognitiveservices/v1' + + headers = { + 'Authorization': 'Bearer ' + get_access_token(), + 'Content-Type': 'application/ssml+xml', + 'X-Microsoft-OutputFormat': playback_format + } + + ssml = f'' + ssml += f'' + ssml += text + ssml += '' + ssml += '' + + response = requests.post(url, headers=headers, data=ssml.encode('utf-8')) + + raw_audio, sample_rate = librosa.load(io.BytesIO(response.content), sr=48000) + resampled = librosa.resample(raw_audio, sample_rate, 44100) + + output_buffer = io.BytesIO() + sf.write(output_buffer, resampled, 44100, 'PCM_16', format='wav') + output_buffer.seek(0) + + return func.HttpResponse(output_buffer.read(), status_code=200) diff --git a/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/text-to-speech/function.json b/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/text-to-speech/function.json new file mode 100644 index 00000000..d9019652 --- /dev/null +++ b/6-consumer/lessons/3-spoken-feedback/code-spoken-response/functions/smart-timer-trigger/text-to-speech/function.json @@ -0,0 +1,20 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "function", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get", + "post" + ] + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] +} \ No newline at end of file diff --git a/6-consumer/lessons/3-spoken-feedback/code-timer/wio-terminal/smart-timer/src/language_understanding.h b/6-consumer/lessons/3-spoken-feedback/code-timer/wio-terminal/smart-timer/src/language_understanding.h index 9d6227a0..1c8d8653 100644 --- a/6-consumer/lessons/3-spoken-feedback/code-timer/wio-terminal/smart-timer/src/language_understanding.h +++ b/6-consumer/lessons/3-spoken-feedback/code-timer/wio-terminal/smart-timer/src/language_understanding.h @@ -16,8 +16,7 @@ public: doc["text"] = text; String body; - JsonObject obj = doc.as(); - serializeJson(obj, body); + serializeJson(doc, body); HTTPClient httpClient; httpClient.begin(_client, TEXT_TO_TIMER_FUNCTION_URL); diff --git a/6-consumer/lessons/3-spoken-feedback/wio-terminal-set-timer.md b/6-consumer/lessons/3-spoken-feedback/wio-terminal-set-timer.md index 1a1b4299..04026ad0 100644 --- a/6-consumer/lessons/3-spoken-feedback/wio-terminal-set-timer.md +++ b/6-consumer/lessons/3-spoken-feedback/wio-terminal-set-timer.md @@ -72,8 +72,7 @@ Microcontrollers don't natively have support for multiple threads in Arduino, so doc["text"] = text; String body; - JsonObject obj = doc.as(); - serializeJson(obj, body); + serializeJson(doc, body); ``` This coverts the text passed to the `GetTimerDuration` method into the following JSON: diff --git a/6-consumer/lessons/3-spoken-feedback/wio-terminal-text-to-speech.md b/6-consumer/lessons/3-spoken-feedback/wio-terminal-text-to-speech.md index 133d5f1d..d7cd8725 100644 --- a/6-consumer/lessons/3-spoken-feedback/wio-terminal-text-to-speech.md +++ b/6-consumer/lessons/3-spoken-feedback/wio-terminal-text-to-speech.md @@ -44,7 +44,7 @@ Instead of downloading and decoding this entire list on your microcontroller, yo 1. Open the `local.settings.json` file and add settings for the speech API key and location: ```json - "SPEECH_KEY": "`", + "SPEECH_KEY": "", "SPEECH_LOCATION": "" ``` @@ -103,6 +103,8 @@ Instead of downloading and decoding this entire list on your microcontroller, yo Replace `` with your language, such as `en-GB`, or `zh-CN`. +> 💁 You can find this code in the [code-spoken-response/functions](code-spoken-response/functions) folder. + ### Task - retrieve the voice from your Wio Terminal 1. Open the `smart-timer` project in VS Code if it is not already open. @@ -190,8 +192,7 @@ Instead of downloading and decoding this entire list on your microcontroller, yo doc["language"] = LANGUAGE; String body; - JsonObject obj = doc.as(); - serializeJson(obj, body); + serializeJson(doc, body); ``` 1. Next create an `HTTPClient`, then use it to call the functions app to get the voices, posting the JSON document: @@ -397,6 +398,8 @@ When needing to manipulate data like this, it is often better to use serverless This will save the audio to `hello.wav` in the current directory. +> 💁 You can find this code in the [code-spoken-response/functions](code-spoken-response/functions) folder. + ### Task - retrieve the speech from your Wio Terminal 1. Open the `smart-timer` project in VS Code if it is not already open. @@ -426,8 +429,7 @@ When needing to manipulate data like this, it is often better to use serverless doc["text"] = text; String body; - JsonObject obj = doc.as(); - serializeJson(obj, body); + serializeJson(doc, body); ``` This writes the language, voice and text to the JSON document, then serializes it to a string. @@ -469,44 +471,19 @@ When needing to manipulate data like this, it is often better to use serverless ### Task - play audio from your Wio Terminal - - - - - - - - - - - - - - +**Coming soon** ## Deploying your functions app to the cloud +The reason for running the functions app locally is because the `librosa` Pip package on linux has a dependency on a library that is not installed by default, and will need to be installed before the function app can run. Function apps are serverless - there are no servers you can manage yourself, so no way to install this library up front. - #include - - +The way to do this is instead to deploy your functions app using a Docker container. This container is deployed by the cloud whenever it needs to spin up a new instance of your function app (such as when the demand exceeds the available resources, or if the function app hasn't been used for a while and is closed down). - * If you are running the function app in the cloud, add the following to the `private` section of the class: +You can find the instructions to set up a function app and deploy via Docker in the [create a function on Linux using a custom container documentation on Microsoft Docs](https://docs.microsoft.com/azure/azure-functions/functions-create-function-linux-custom-image?tabs=bash%2Cazurecli&pivots=programming-language-python&WT.mc_id=academic-17441-jabenn). - ```cpp - WiFiClientSecure _client; - ``` - - You will also need to set the certificate on this class, so add the following constructor to the `public` section: - - ```cpp - LanguageUnderstanding() - { - _client.setCACert(FUNCTIONS_CERTIFICATE); - } - ``` +Once this has been deployed, you can port your Wio Terminal code to access this function: -1. If you have deployed your functions app to the cloud, add the following certificate for `azurewebsites.net` to the `config.h` file. +1. Add the Azure Functions certificate to `config.h`: ```cpp const char *FUNCTIONS_CERTIFICATE = @@ -543,4 +520,12 @@ When needing to manipulate data like this, it is often better to use serverless "-----END CERTIFICATE-----\r\n"; ``` - > 💁 If you are accessing your functions app locally, you don't need to do this. \ No newline at end of file +1. Change all includes of `` to ``. + +1. Change all `WiFiClient` fields to `WiFiClientSecure`. + +1. In every class that has a `WiFiClientSecure` field, add a constructor and set the certificate in that constructor: + + ```cpp + _client.setCACert(FUNCTIONS_CERTIFICATE); + ``` diff --git a/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/get-voices/__init__.py b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/get-voices/__init__.py new file mode 100644 index 00000000..44c81900 --- /dev/null +++ b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/get-voices/__init__.py @@ -0,0 +1,26 @@ +import json +import os +import requests + +import azure.functions as func + +def main(req: func.HttpRequest) -> func.HttpResponse: + location = os.environ['SPEECH_LOCATION'] + speech_key = os.environ['SPEECH_KEY'] + + req_body = req.get_json() + language = req_body['language'] + + url = f'https://{location}.tts.speech.microsoft.com/cognitiveservices/voices/list' + + headers = { + 'Ocp-Apim-Subscription-Key': speech_key + } + + response = requests.get(url, headers=headers) + voices_json = json.loads(response.text) + + voices = filter(lambda x: x['Locale'].lower() == language.lower(), voices_json) + voices = map(lambda x: x['ShortName'], voices) + + return func.HttpResponse(json.dumps(list(voices)), status_code=200) diff --git a/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/get-voices/function.json b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/get-voices/function.json new file mode 100644 index 00000000..d9019652 --- /dev/null +++ b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/get-voices/function.json @@ -0,0 +1,20 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "function", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get", + "post" + ] + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] +} \ No newline at end of file diff --git a/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/host.json b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/host.json new file mode 100644 index 00000000..291065f8 --- /dev/null +++ b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/host.json @@ -0,0 +1,15 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[2.*, 3.0.0)" + } +} \ No newline at end of file diff --git a/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/local.settings.json b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/local.settings.json new file mode 100644 index 00000000..a88a77ff --- /dev/null +++ b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/local.settings.json @@ -0,0 +1,14 @@ +{ + "IsEncrypted": false, + "Values": { + "FUNCTIONS_WORKER_RUNTIME": "python", + "AzureWebJobsStorage": "", + "LUIS_KEY": "", + "LUIS_ENDPOINT_URL": "", + "LUIS_APP_ID": "", + "SPEECH_KEY": "", + "SPEECH_LOCATION": "", + "TRANSLATOR_KEY": "", + "TRANSLATOR_LOCATION": "" + } +} \ No newline at end of file diff --git a/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/requirements.txt b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/requirements.txt new file mode 100644 index 00000000..a2596be3 --- /dev/null +++ b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/requirements.txt @@ -0,0 +1,5 @@ +# Do not include azure-functions-worker as it may conflict with the Azure Functions platform + +azure-functions +azure-cognitiveservices-language-luis +librosa \ No newline at end of file diff --git a/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/text-to-speech/__init__.py b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/text-to-speech/__init__.py new file mode 100644 index 00000000..f09404f3 --- /dev/null +++ b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/text-to-speech/__init__.py @@ -0,0 +1,52 @@ +import io +import os +import requests + +import librosa +import soundfile as sf +import azure.functions as func + +location = os.environ['SPEECH_LOCATION'] +speech_key = os.environ['SPEECH_KEY'] + +def get_access_token(): + headers = { + 'Ocp-Apim-Subscription-Key': speech_key + } + + token_endpoint = f'https://{location}.api.cognitive.microsoft.com/sts/v1.0/issuetoken' + response = requests.post(token_endpoint, headers=headers) + return str(response.text) + +playback_format = 'riff-48khz-16bit-mono-pcm' + +def main(req: func.HttpRequest) -> func.HttpResponse: + req_body = req.get_json() + language = req_body['language'] + voice = req_body['voice'] + text = req_body['text'] + + url = f'https://{location}.tts.speech.microsoft.com/cognitiveservices/v1' + + headers = { + 'Authorization': 'Bearer ' + get_access_token(), + 'Content-Type': 'application/ssml+xml', + 'X-Microsoft-OutputFormat': playback_format + } + + ssml = f'' + ssml += f'' + ssml += text + ssml += '' + ssml += '' + + response = requests.post(url, headers=headers, data=ssml.encode('utf-8')) + + raw_audio, sample_rate = librosa.load(io.BytesIO(response.content), sr=48000) + resampled = librosa.resample(raw_audio, sample_rate, 44100) + + output_buffer = io.BytesIO() + sf.write(output_buffer, resampled, 44100, 'PCM_16', format='wav') + output_buffer.seek(0) + + return func.HttpResponse(output_buffer.read(), status_code=200) diff --git a/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/text-to-speech/function.json b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/text-to-speech/function.json new file mode 100644 index 00000000..d9019652 --- /dev/null +++ b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/text-to-speech/function.json @@ -0,0 +1,20 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "function", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get", + "post" + ] + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] +} \ No newline at end of file diff --git a/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/text-to-timer/__init__.py b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/text-to-timer/__init__.py new file mode 100644 index 00000000..d15d6e68 --- /dev/null +++ b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/text-to-timer/__init__.py @@ -0,0 +1,46 @@ +import logging + +import azure.functions as func +import json +import os +from azure.cognitiveservices.language.luis.runtime import LUISRuntimeClient +from msrest.authentication import CognitiveServicesCredentials + + +def main(req: func.HttpRequest) -> func.HttpResponse: + luis_key = os.environ['LUIS_KEY'] + endpoint_url = os.environ['LUIS_ENDPOINT_URL'] + app_id = os.environ['LUIS_APP_ID'] + + credentials = CognitiveServicesCredentials(luis_key) + client = LUISRuntimeClient(endpoint=endpoint_url, credentials=credentials) + + req_body = req.get_json() + text = req_body['text'] + logging.info(f'Request - {text}') + prediction_request = { 'query' : text } + + prediction_response = client.prediction.get_slot_prediction(app_id, 'Staging', prediction_request) + + if prediction_response.prediction.top_intent == 'set timer': + numbers = prediction_response.prediction.entities['number'] + time_units = prediction_response.prediction.entities['time unit'] + total_seconds = 0 + + for i in range(0, len(numbers)): + number = numbers[i] + time_unit = time_units[i][0] + + if time_unit == 'minute': + total_seconds += number * 60 + else: + total_seconds += number + + logging.info(f'Timer required for {total_seconds} seconds') + + payload = { + 'seconds': total_seconds + } + return func.HttpResponse(json.dumps(payload), status_code=200) + + return func.HttpResponse(status_code=404) \ No newline at end of file diff --git a/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/text-to-timer/function.json b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/text-to-timer/function.json new file mode 100644 index 00000000..d9019652 --- /dev/null +++ b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/text-to-timer/function.json @@ -0,0 +1,20 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "function", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get", + "post" + ] + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] +} \ No newline at end of file diff --git a/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/translate-text/__init__.py b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/translate-text/__init__.py new file mode 100644 index 00000000..850200a6 --- /dev/null +++ b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/translate-text/__init__.py @@ -0,0 +1,36 @@ +import logging +import os +import requests + +import azure.functions as func + +location = os.environ['TRANSLATOR_LOCATION'] +translator_key = os.environ['TRANSLATOR_KEY'] + +def main(req: func.HttpRequest) -> func.HttpResponse: + req_body = req.get_json() + from_language = req_body['from_language'] + to_language = req_body['to_language'] + text = req_body['text'] + + logging.info(f'Translating {text} from {from_language} to {to_language}') + + url = f'https://api.cognitive.microsofttranslator.com/translate?api-version=3.0' + + headers = { + 'Ocp-Apim-Subscription-Key': translator_key, + 'Ocp-Apim-Subscription-Region': location, + 'Content-type': 'application/json' + } + + params = { + 'from': from_language, + 'to': to_language + } + + body = [{ + 'text' : text + }] + + response = requests.post(url, headers=headers, params=params, json=body) + return func.HttpResponse(response.json()[0]['translations'][0]['text']) diff --git a/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/translate-text/__pycache__/__init__.cpython-39.pyc b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/translate-text/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 00000000..1e106428 Binary files /dev/null and b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/translate-text/__pycache__/__init__.cpython-39.pyc differ diff --git a/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/translate-text/function.json b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/translate-text/function.json new file mode 100644 index 00000000..d9019652 --- /dev/null +++ b/6-consumer/lessons/4-multiple-language-support/code/functions/smart-timer-trigger/translate-text/function.json @@ -0,0 +1,20 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "function", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get", + "post" + ] + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] +} \ No newline at end of file diff --git a/6-consumer/lessons/4-multiple-language-support/pi-translate-speech.md b/6-consumer/lessons/4-multiple-language-support/pi-translate-speech.md index 51305971..59c08ff5 100644 --- a/6-consumer/lessons/4-multiple-language-support/pi-translate-speech.md +++ b/6-consumer/lessons/4-multiple-language-support/pi-translate-speech.md @@ -8,7 +8,7 @@ The speech service REST API doesn't support direct translations, instead you can ### Task - use the translator resource to translate text -1. Your smart timer will have 2 languages set - the language of the server that was used to train LUIS, and the language spoken by the user. Update the `language` variable to be the language that will be spoken by the used, and add a new variable called `server_language` for the language used to train LUIS: +1. Your smart timer will have 2 languages set - the language of the server that was used to train LUIS (the same language is also used to build the messages to speak to the user), and the language spoken by the user. Update the `language` variable to be the language that will be spoken by the user, and add a new variable called `server_language` for the language used to train LUIS: ```python language = '' diff --git a/6-consumer/lessons/4-multiple-language-support/virtual-device-translate-speech.md b/6-consumer/lessons/4-multiple-language-support/virtual-device-translate-speech.md index 15db3690..493a5b40 100644 --- a/6-consumer/lessons/4-multiple-language-support/virtual-device-translate-speech.md +++ b/6-consumer/lessons/4-multiple-language-support/virtual-device-translate-speech.md @@ -20,7 +20,7 @@ The speech service can take speech and not only convert to text in the same lang This imports classes used to translate speech, and a `requests` library that will be used to make a call to the Translator service later in this lesson. -1. Your smart timer will have 2 languages set - the language of the server that was used to train LUIS, and the language spoken by the user. Update the `language` variable to be the language that will be spoken by the used, and add a new variable called `server_language` for the language used to train LUIS: +1. Your smart timer will have 2 languages set - the language of the server that was used to train LUIS (the same language is also used to build the messages to speak to the user), and the language spoken by the user. Update the `language` variable to be the language that will be spoken by the user, and add a new variable called `server_language` for the language used to train LUIS: ```python language = '' diff --git a/6-consumer/lessons/4-multiple-language-support/wio-terminal-translate-speech.md b/6-consumer/lessons/4-multiple-language-support/wio-terminal-translate-speech.md index 11a7085d..2e3ce926 100644 --- a/6-consumer/lessons/4-multiple-language-support/wio-terminal-translate-speech.md +++ b/6-consumer/lessons/4-multiple-language-support/wio-terminal-translate-speech.md @@ -4,13 +4,91 @@ In this part of the lesson, you will write code to translate text using the tran ## Convert text to speech using the translator service -The speech service REST API doesn't support direct translations, instead you can use the Translator service to translate the text generated by the speech to text service, and the text of the spoken response. This service has a REST API you can use to translate the text. +The speech service REST API doesn't support direct translations, instead you can use the Translator service to translate the text generated by the speech to text service, and the text of the spoken response. This service has a REST API you can use to translate the text, but to make it easier to use this will be wrapped in another HTTP trigger in your functions app. -### Task - use the translator resource to translate text +### Task - create a serverless function to translate text + +1. Open your `smart-timer-trigger` project in VS Code, and open the terminal ensuring the virtual environment is activated. If not, kill and re-create the terminal. + +1. Open the `local.settings.json` file and add settings for the translator API key and location: + + ```json + "TRANSLATOR_KEY": "", + "TRANSLATOR_LOCATION": "" + ``` + + Replace `` with the API key for your translator service resource. Replace `` with the location you used when you created the translator service resource. + +1. Add a new HTTP trigger to this app called `translate-text` using the following command from inside the VS Code terminal in the root folder of the functions app project: + + ```sh + func new --name translate-text --template "HTTP trigger" + ``` + + This will create an HTTP trigger called `translate-text`. + +1. Replace the contents of the `__init__.py` file in the `translate-text` folder with the following: + + ```python + import logging + import os + import requests + + import azure.functions as func + + location = os.environ['TRANSLATOR_LOCATION'] + translator_key = os.environ['TRANSLATOR_KEY'] + + def main(req: func.HttpRequest) -> func.HttpResponse: + req_body = req.get_json() + from_language = req_body['from_language'] + to_language = req_body['to_language'] + text = req_body['text'] + + logging.info(f'Translating {text} from {from_language} to {to_language}') + + url = f'https://api.cognitive.microsofttranslator.com/translate?api-version=3.0' + + headers = { + 'Ocp-Apim-Subscription-Key': translator_key, + 'Ocp-Apim-Subscription-Region': location, + 'Content-type': 'application/json' + } + + params = { + 'from': from_language, + 'to': to_language + } + + body = [{ + 'text' : text + }] + + response = requests.post(url, headers=headers, params=params, json=body) + return func.HttpResponse(response.json()[0]['translations'][0]['text']) + ``` + + This code extracts the text and the languages from the HTTP request. It then makes a request to the translator REST API, passing the languages as parameters for the URL and the text to translate as the body. Finally, the translation is returned. + +1. Run your function app locally. You can then call this using a tool like curl in the same way that you tested your `text-to-timer` HTTP trigger. Make sure to pass the text to translate and the languages as a JSON body: + + ```json + { + "text": "Définir une minuterie de 30 secondes", + "from_language": "fr-FR", + "to_language": "en-US" + } + ``` + + This example translates *Définir une minuterie de 30 secondes* from French to US English. It will return *Set a 30-second timer*. + +> 💁 You can find this code in the [code/functions](code/functions) folder. + +### Task - use the translator function to translate text 1. Open the `smart-timer` project in VS Code if it is not already open. -1. Your smart timer will have 2 languages set - the language of the server that was used to train LUIS, and the language spoken by the user. Update the `LANGUAGE` constant in the `config.h` header file to be the language that will be spoken by the used, and add a new constant called `SERVER_LANGUAGE` for the language used to train LUIS: +1. Your smart timer will have 2 languages set - the language of the server that was used to train LUIS (the same language is also used to build the messages to speak to the user), and the language spoken by the user. Update the `LANGUAGE` constant in the `config.h` header file to be the language that will be spoken by the user, and add a new constant called `SERVER_LANGUAGE` for the language used to train LUIS: ```cpp const char *LANGUAGE = ""; @@ -29,153 +107,164 @@ The speech service REST API doesn't support direct translations, instead you can > > ![The listen translation button on Bing translate](../../../images/bing-translate.png) -1. Add the translator API key below the `SPEECH_API_KEY`: +1. Add the translator API key and location below the `SPEECH_LOCATION`: ```cpp - const char *TRANSLATOR_API_KEY = ""; + const char *TRANSLATOR_API_KEY = ""; + const char *TRANSLATOR_LOCATION = ""; ``` - Replace `` with the API key for your translator service resource. + Replace `` with the API key for your translator service resource. Replace `` with the location you used when you created the translator service resource. -1. Add the translator URL below the `VOICE_URL`: +1. Add the translator trigger URL below the `VOICE_URL`: ```cpp - const char *TRANSLATE_URL = "https://api.cognitive.microsofttranslator.com/translate?api-version=3.0&from=%s&to=%s"; + const char *TRANSLATE_FUNCTION_URL = ""; ``` - This is a global URL and doesn't need your location in it, instead the location of the translator resource is passed as a header. This URL will need the from and to languages - so the language that the text is in and the language you want it translated to. - - - - - - - - - - - - - + Replace `` with the URL for the `translate-text` HTTP trigger on your function app. This will be the same as the value for `TEXT_TO_TIMER_FUNCTION_URL`, except with a function name of `translate-text` instead of `text-to-timer`. +1. Add a new file to the `src` folder called `text_translator.h`. +1. This new `text_translator.h` header file will contain a class to translate text. Add the following to this file to declare this class: + ```cpp + #pragma once + + #include + #include + #include + #include + + #include "config.h" + + class TextTranslator + { + public: + private: + WiFiClient _client; + }; + + TextTranslator textTranslator; + ``` + This declares the `TextTranslator` class, along with an instance of this class. The class has a single field for the WiFi client. +1. To the `public` section of this class, add a method to translate text: + ```cpp + String translateText(String text, String from_language, String to_language) + { + } + ``` + This method takes the language to translate from, and the language to translate to. When handling speech, the speech will be translated from the user language to the LUIS server language, and when giving responses it will translate from the LUIS server language to the users language. -1. Above the `say` function, define a `translate_text` function that will translate text from the server language to the user language: +1. In this method, add code to construct a JSON body containing the text to translate and the languages: - ```python - def translate_text(text, from_language, to_language): + ```cpp + DynamicJsonDocument doc(1024); + doc["text"] = text; + doc["from_language"] = from_language; + doc["to_language"] = to_language; + + String body; + serializeJson(doc, body); + + Serial.print("Translating "); + Serial.print(text); + Serial.print(" from "); + Serial.print(from_language); + Serial.print(" to "); + Serial.print(to_language); ``` - The from and to languages are passed to this function - your app needs to convert from user language to server language when recognizing speech, and from server language to user language when provided spoken feedback. +1. Below this, add the following code to send the body to the serverless function app: -1. Inside this function, define the URL and headers for the REST API call: - - ```python - url = f'https://api.cognitive.microsofttranslator.com/translate?api-version=3.0' + ```cpp + HTTPClient httpClient; + httpClient.begin(_client, TRANSLATE_FUNCTION_URL); - headers = { - 'Ocp-Apim-Subscription-Key': translator_api_key, - 'Ocp-Apim-Subscription-Region': location, - 'Content-type': 'application/json' - } + int httpResponseCode = httpClient.POST(body); ``` - The URL for this API is not location specific, instead the location is passed in as a header. The API key is used directly, so unlike the speech service there is no need to get an access token from the token issuer API. +1. Next, add code to get the response: -1. Below this define the parameters and body for the call: + ```cpp + String translated_text = ""; - ```python - params = { - 'from': from_language, - 'to': to_language + if (httpResponseCode == 200) + { + translated_text = httpClient.getString(); + Serial.print("Translated: "); + Serial.println(translated_text); + } + else + { + Serial.print("Failed to translate text - error "); + Serial.println(httpResponseCode); } - - body = [{ - 'text' : text - }] ``` - The `params` defines the parameters to pass to the API call, passing the from and to languages. This call will translate text in the `from` language into the `to` language. - - The `body` contains the text to translate. This is an array, as multiple blocks of text can be translated in the same call. +1. Finally, add code to close the connection and return the translated text: -1. Make the call the REST API, and get the response: + ```cpp + httpClient.end(); - ```python - response = requests.post(url, headers=headers, params=params, json=body) + return translated_text; ``` - The response that comes back is a JSON array, with one item that contains the translations. This item has an array for translations of all the items passed in the body. +### Task - translate the recognized speech and the responses - ```json - [ - { - "translations": [ - { - "text": "Set a 2 minute 27 second timer.", - "to": "en" - } - ] - } - ] - ``` +1. Open the `main.cpp` file. -1. Return the `test` property from the first translation from the first item in the array: +1. Add an include directive at the top of the file for the `TextTranslator` class header file: - ```python - return response.json()[0]['translations'][0]['text'] + ```cpp + #include "text_translator.h" ``` -1. Update the `while True` loop to translate the text from the call to `convert_speech_to_text` from the user language to the server language: - - ```python - if len(text) > 0: - print('Original:', text) - text = translate_text(text, language, server_language) - print('Translated:', text) +1. The text that is said when a timer is set or expires needs to be translated. To do this, add the following as the first line of the `say` function: - message = Message(json.dumps({ 'speech': text })) - device_client.send_message(message) + ```cpp + text = textTranslator.translateText(text, LANGUAGE, SERVER_LANGUAGE); ``` - This code also prints the original and translated versions of the text to the console. + This will translate the text to the users language. -1. Update the `say` function to translate the text to say from the server language to the user language: +1. In the `processAudio` function, text is retrieved from the captured audio with the `String text = speechToText.convertSpeechToText();` call. After this call, translate the text: - ```python - def say(text): - print('Original:', text) - text = translate_text(text, server_language, language) - print('Translated:', text) - speech = get_speech(text) - play_speech(speech) + ```cpp + String text = speechToText.convertSpeechToText(); + text = textTranslator.translateText(text, LANGUAGE, SERVER_LANGUAGE); ``` - This code also prints the original and translated versions of the text to the console. + This will translate the text from the users language into the language used on the server. -1. Run your code. Ensure your function app is running, and request a timer in the user language, either by speaking that language yourself, or using a translation app. +1. Build this code, upload it to your Wio Terminal and test it out through the serial monitor. Once you see `Ready` in the serial monitor, press the C button (the one on the left-hand side, closest to the power switch), and speak. Ensure your function app is running, and request a timer in the user language, either by speaking that language yourself, or using a translation app. ```output - pi@raspberrypi:~/smart-timer $ python3 app.py - Connecting - Connected - Using voice fr-FR-DeniseNeural - Original: Définir une minuterie de 2 minutes et 27 secondes. - Translated: Set a timer of 2 minutes and 27 seconds. - Original: 2 minute 27 second timer started. + Connecting to WiFi.. + Connected! + Got access token. + Ready. + Starting recording... + Finished recording + Sending speech... + Speech sent! + {"RecognitionStatus":"Success","DisplayText":"Définir une minuterie de 2 minutes 27 secondes.","Offset":9600000,"Duration":40400000} + Translating Définir une minuterie de 2 minutes 27 secondes. from fr-FR to en-US + Translated: Set a timer of 2 minutes 27 seconds. + Set a timer of 2 minutes 27 seconds. + {"seconds": 147} + Translating 2 minute 27 second timer started. from en-US to fr-FR Translated: 2 minute 27 seconde minute a commencé. - Original: Times up on your 2 minute 27 second timer. + 2 minute 27 seconde minute a commencé. + Translating Times up on your 2 minute 27 second timer. from en-US to fr-FR Translated: Chronométrant votre minuterie de 2 minutes 27 secondes. + Chronométrant votre minuterie de 2 minutes 27 secondes. ``` - > 💁 Due to the different ways of saying something in different languages, you may get translations that are slightly different to the examples you gave LUIS. If this is the case, add more examples to LUIS, retrain then re-publish the model. - -> 💁 You can find this code in the [code/pi](code/pi) folder. +> 💁 You can find this code in the [code/wio-terminal](code/wio-terminal) folder. 😀 Your multi-lingual timer program was a success! -