My name is Cobi Christiansen and I am the Data Operations Specialist at Coastal Carolina University. I am working on integrating a YSI EXO2 water quality sonde that has the following sensors: temperature, conductivity, dissolved oxygen, pH, and turbidity. The physical integration is nearly complete, but I am still working on developing an application that parses the incoming sonde data, aggregates data over a period of time, calculates min, max, mean, and standard deviation over aggregation period, converts stats to number format, and finally sends data to the bristlemouth network. I am trying to modify the existing code from src/apps/bm_devkit/rbr_coda_example. I have some coding experience, but C++ is new to me.
I am struggling to get the parser to work with my line of data (outlined at the top of code below). I reached out to YSI to see if the sonde can spit out just the data values without the #, but I have not heard back yet.
Any assistance with this code would be greatly appreciated!
/*
* This is a lightweight example Dev Kit application (v.0.6.0) to integrate a YSI EXO2 Sonde.
* Sonde Paramters: temp, conductivity, DO, pH
* Example line: # 21.853 0.790 0.00 7.58
* Current Issue 1: Parsing the "#" in the line... "ERR - can't parse value of ValueType 0"
Solution?: Find the correct way to parse the "#" or find a way to exclude the first character when parsing
* Current Issue 2: It does not seem to be aggregating the data correctly
Solution?: Check to make sure exo_data is pulling correctly. Does this have to do with the parsing issue?
*/
#include "user_code.h"
#include "LineParser.h"
#include "OrderedSeparatorLineParser.h"
#include "array_utils.h"
#include "avgSampler.h"
#include "bm_network.h"
#include "bm_printf.h"
#include "bm_pubsub.h"
#include "bristlefin.h"
#include "bsp.h"
#include "debug.h"
#include "lwip/inet.h"
#include "payload_uart.h"
#include "sensors.h"
#include "stm32_rtc.h"
#include "task_priorities.h"
#include "uptime.h"
#include "usart.h"
#include "util.h"
#define LED_ON_TIME_MS 20
#define LED_PERIOD_MS 1000
#define DEFAULT_BAUD_RATE 9600
#define DEFAULT_LINE_TERM 13 // A command is terminated by a <CR>
#define BYTES_CLUSTER_MS 50
// How often to compute and return statistics
#define EXO_AGG_PERIOD_MIN 0.5
// 10 min => 600,000 ms
#define EXO_AGG_PERIOD_MS (EXO_AGG_PERIOD_MIN * 60 * 1000)
// We have enough RAM that we can keep it simple for shorter durations - use 64 bit doubles, buffer all readings.
// We could be much more RAM and precision efficient by using numerical methods like Kahan summation and Welford's algorithm.*/
#define MAX_EXO_SAMPLES ((EXO_AGG_PERIOD_MS / 1000) + 10)
typedef struct {
uint16_t sample_count;
double min;
double max;
double mean;
double stdev;
} __attribute__((__packed__)) exoData_t;
#define EXO_DATA_SIZE sizeof(exoData_t)
// Create an instance of the averaging sampler for our data
static AveragingSampler exo_data;
// app_main passes a handle to the user config partition in NVM.
extern cfg::Configuration *userConfigurationPartition;
// A timer variable we can set to trigger a pulse on LED2 when we get payload serial data
static int32_t ledLinePulse = -1;
static u_int32_t baud_rate_config = DEFAULT_BAUD_RATE;
static u_int32_t line_term_config = DEFAULT_LINE_TERM;
// A buffer for our data from the payload uart
char payload_buffer[2048];
/* For Turning Text Into Numbers...
* Setup a LineParser to turn the ASCII serial data from the YSI EXO2 sonde into numbers
* Lines from the EXO2 look like: # 21.853 0.790 0.00 7.58
* 256 character buffer should be more than enough
* For unsigned ints, let's use 64 bits, and for floating point let's use 64 bit doubles.
* We've got luxurious amounts of RAM on this chip, and it's much easier to avoid roll-overs and precision issues
* by using it vs. troubleshooting them because we prematurely optimized things.
* Declare the parser here with separator, buffer length, value types array, and number of values per line.
* We'll initialize the parser later in setup to allocate all the memory we'll need.
*/
ValueType valueTypes[] = {TYPE_INVALID, TYPE_DOUBLE, TYPE_DOUBLE, TYPE_DOUBLE, TYPE_DOUBLE};
OrderedSeparatorLineParser parser(" ", 256, valueTypes, 5);
void setup(void) {
/* USER ONE-TIME SETUP CODE GOES HERE */
// Initialize the exo data buffer to be the size we need.
exo_data.initBuffer(MAX_EXO_SAMPLES);
// Initialize our LineParser, which will allocated any needed memory for parsing.
parser.init();
// Setup the UART – the on-board serial driver that talks to the RS232 transceiver.
PLUART::init(USER_TASK_PRIORITY);
// Baud set per expected baud rate of the sensor.
PLUART::setBaud(baud_rate_config);
// Enable passing raw bytes to user app.
PLUART::setUseByteStreamBuffer(true);
// Enable parsing lines and passing to user app.
/// Warning: PLUART only stores a single line at a time. If your attached payload sends lines
/// faster than the app reads them, they will be overwritten and data will be lost.
PLUART::setUseLineBuffer(true);
// Set a line termination character per protocol of the sensor.
PLUART::setTerminationCharacter((char)line_term_config);
// Turn on the UART.
PLUART::enable();
// Enable the input to the Vout power supply.
bristlefin.enableVbus();
// ensure Vbus stable before enable Vout with a 5ms delay.
vTaskDelay(pdMS_TO_TICKS(5));
// enable Vout, 12V by default.
bristlefin.enableVout();
}
void loop(void) {
/* USER LOOP CODE GOES HERE */
// This aggregates exo readings into stats, and sends them along to Spotter
static u_int32_t exoStatsTimer = uptimeGetMs();
if ((u_int32_t)uptimeGetMs() - exoStatsTimer >= EXO_AGG_PERIOD_MS) {
exoStatsTimer = uptimeGetMs();
double mean = 0, stdev = 0, min = 0, max = 0;
uint16_t n_samples = 0;
if (exo_data.getNumSamples()) {
mean = exo_data.getMean();
stdev = exo_data.getStd(mean);
min = exo_data.getMin();
max = exo_data.getMax();
n_samples = exo_data.getNumSamples();
exo_data.clear();
}
// Get the RTC if available
RTCTimeAndDate_t time_and_date = {};
rtcGet(&time_and_date);
char rtcTimeBuffer[32];
rtcPrint(rtcTimeBuffer, &time_and_date);
bm_fprintf(0, "payload_data_agg.log",
"tick: %llu, rtc: %s, n: %u, min: %.4f, max: %.4f, mean: %.4f, "
"std: %.4f\n",
uptimeGetMs(), rtcTimeBuffer, n_samples, min, max, mean, stdev);
bm_printf(0,
"[exo-agg] | tick: %llu, rtc: %s, n: %u, min: %.4f, max: %.4f, "
"mean: %.4f, std: %.4f",
uptimeGetMs(), rtcTimeBuffer, n_samples, min, max, mean, stdev);
printf("[exo-agg] | tick: %llu, rtc: %s, n: %u, min: %.4f, max: %.4f, "
"mean: %.4f, std: %.4f\n",
uptimeGetMs(), rtcTimeBuffer, n_samples, min, max, mean, stdev);
uint8_t tx_data[EXO_DATA_SIZE] = {};
exoData_t tx_exo = {
.sample_count = n_samples, .min = min, .max = max, .mean = mean, .stdev = stdev};
memcpy(tx_data, (uint8_t *)(&tx_exo), EXO_DATA_SIZE);
if (spotter_tx_data(tx_data, EXO_DATA_SIZE, BM_NETWORK_TYPE_CELLULAR_IRI_FALLBACK)) {
printf("%llut - %s | Sucessfully sent Spotter transmit data request\n", uptimeGetMs(),
rtcTimeBuffer);
} else {
printf("%llut - %s | Failed to send Spotter transmit data request\n", uptimeGetMs(),
rtcTimeBuffer);
}
}
// This checks for a trigger set by ledLinePulse when data is received from the payload UART.
// Each time this happens, we pulse LED2 Green.
static bool led2State = false;
// If LED2 is off and the ledLinePulse flag is set, turn it on Green.
if (!led2State && ledLinePulse > -1) {
bristlefin.setLed(2, Bristlefin::LED_GREEN);
led2State = true;
}
// If LED2 has been on for LED_ON_TIME_MS, turn it off.
else if (led2State && ((u_int32_t)uptimeGetMs() - ledLinePulse >= LED_ON_TIME_MS)) {
bristlefin.setLed(2, Bristlefin::LED_OFF);
ledLinePulse = -1;
led2State = false;
}
// This section demonstrates a simple non-blocking bare metal method for rollover-safe timed tasks, like blinking an LED.
// More canonical (but more arcane) modern methods of implementing this kind functionality would bee to use FreeRTOS tasks or hardware timer ISRs.
static u_int32_t ledPulseTimer = uptimeGetMs();
static u_int32_t ledOnTimer = 0;
static bool led1State = false;
// Turn LED1 on green every LED_PERIOD_MS milliseconds.
if (!led1State && ((u_int32_t)uptimeGetMs() - ledPulseTimer >= LED_PERIOD_MS)) {
bristlefin.setLed(1, Bristlefin::LED_GREEN);
ledOnTimer = uptimeGetMs();
ledPulseTimer += LED_PERIOD_MS;
led1State = true;
}
// If LED1 has been on for LED_ON_TIME_MS milliseconds, turn it off.
else if (led1State && ((u_int32_t)uptimeGetMs() - ledOnTimer >= LED_ON_TIME_MS)) {
bristlefin.setLed(1, Bristlefin::LED_OFF);
led1State = false;
}
// Read a cluster of bytes if available
// A timer is used to try to keep clusters of bytes (say from lines) in the same output.
static int64_t readingBytesTimer = -1;
// Note - PLUART::setUseByteStreamBuffer must be set true in setup to enable bytes.
if (readingBytesTimer == -1 && PLUART::byteAvailable()) {
// Get the RTC if available
RTCTimeAndDate_t time_and_date = {};
rtcGet(&time_and_date);
char rtcTimeBuffer[32];
rtcPrint(rtcTimeBuffer, &time_and_date);
printf("[payload-bytes] | tick: %" PRIu64 ", rtc: %s, bytes:", uptimeGetMs(),
rtcTimeBuffer);
// not very readable, but it's a compact trick to overload our timer variable with a -1 flag
readingBytesTimer = (int64_t)((u_int32_t)uptimeGetMs());
}
while (PLUART::byteAvailable()) {
readingBytesTimer = (int64_t)((u_int32_t)uptimeGetMs());
uint8_t byte_read = PLUART::readByte();
printf("%02X ", byte_read);
}
if (readingBytesTimer > -1 &&
(u_int32_t)uptimeGetMs() - (u_int32_t)readingBytesTimer >= BYTES_CLUSTER_MS) {
printf("\n");
readingBytesTimer = -1;
}
// Read a line if it is available
if (PLUART::lineAvailable()) {
// Shortcut the raw bytes cluster completion so the parsed line will be on a new console line
if (readingBytesTimer > -1) {
printf("\n");
readingBytesTimer = -1;
}
uint16_t read_len = PLUART::readLine(payload_buffer, sizeof(payload_buffer));
// Get the RTC if available
RTCTimeAndDate_t time_and_date = {};
rtcGet(&time_and_date);
char rtcTimeBuffer[32] = {};
rtcPrint(rtcTimeBuffer, NULL);
bm_fprintf(0, "exo_raw.log", "tick: %" PRIu64 ", rtc: %s, line: %.*s\n", uptimeGetMs(),
rtcTimeBuffer, read_len, payload_buffer);
bm_printf(0, "[exo] | tick: %" PRIu64 ", rtc: %s, line: %.*s", uptimeGetMs(), rtcTimeBuffer,
read_len, payload_buffer);
printf("[exo] | tick: %" PRIu64 ", rtc: %s, line: %.*s\n", uptimeGetMs(), rtcTimeBuffer,
read_len, payload_buffer);
// trigger a pulse on LED2
ledLinePulse = uptimeGetMs();
// Now when we get a line of text data, our LineParser turns it into numeric values.
if (parser.parseLine(payload_buffer, read_len)) {
printf("parsed values: %llu | %f\n", parser.getValue(0).data, parser.getValue(1).data, parser.getValue(2).data), parser.getValue(3).data, parser.getValue(4).data;
} else {
printf("Error parsing line!\n");
return;
}
// Now let's aggregate those values into statistics
if (exo_data.getNumSamples() >= MAX_EXO_SAMPLES) {
printf("ERR - No more room in exo reading buffer, already have %d "
"readings!\n",
MAX_EXO_SAMPLES);
return;
}
double exo_reading = parser.getValue(1).data.double_val;
exo_data.addSample(exo_reading);
printf("count: %u/%d, min: %f, max: %f\n", exo_data.getNumSamples(),
MAX_EXO_SAMPLES, exo_data.getMin(), exo_data.getMax());
}
}
