/**
* This program logs data to a binary file. Functions are included
* to convert the binary file to a csv text file.
*
* Samples are logged at regular intervals. The maximum logging rate
* depends on the quality of your SD card and the time required to
* read sensor data. This example has been tested at 500 Hz with
* good SD card on an Uno. 4000 HZ is possible on a Due.
*
* If your SD card has a long write latency, it may be necessary to use
* slower sample rates. Using a Mega Arduino helps overcome latency
* problems since 12 512 byte buffers will be used.
*
* Data is written to the file using a SD multiple block write command.
*/
#include <SPI.h>
#include "SdFat.h"
#include "FreeStack.h"
#include "UserTypes.h"
#ifdef __AVR_ATmega328P__
#include "MinimumSerial.h"
MinimumSerial MinSerial;
#define Serial MinSerial
#endif // __AVR_ATmega328P__
//==============================================================================
// Start of configuration constants.
//==============================================================================
// Abort run on an overrun. Data before the overrun will be saved.
#define ABORT_ON_OVERRUN 1
//------------------------------------------------------------------------------
//Interval between data records in microseconds.
const uint32_t LOG_INTERVAL_USEC = 2000;
//------------------------------------------------------------------------------
// Set USE_SHARED_SPI non-zero for use of an SPI sensor.
// May not work for some cards.
#ifndef USE_SHARED_SPI
#define USE_SHARED_SPI 0
#endif // USE_SHARED_SPI
//------------------------------------------------------------------------------
// Pin definitions.
//
// SD chip select pin.
const uint8_t SD_CS_PIN = SS;
//
// Digital pin to indicate an error, set to -1 if not used.
// The led blinks for fatal errors. The led goes on solid for
// overrun errors and logging continues unless ABORT_ON_OVERRUN
// is non-zero.
#ifdef ERROR_LED_PIN
#undef ERROR_LED_PIN
#endif // ERROR_LED_PIN
const int8_t ERROR_LED_PIN = -1;
//------------------------------------------------------------------------------
// File definitions.
//
// Maximum file size in blocks.
// The program creates a contiguous file with FILE_BLOCK_COUNT 512 byte blocks.
// This file is flash erased using special SD commands. The file will be
// truncated if logging is stopped early.
const uint32_t FILE_BLOCK_COUNT = 256000;
//
// log file base name if not defined in UserTypes.h
#ifndef FILE_BASE_NAME
#define FILE_BASE_NAME "data"
#endif // FILE_BASE_NAME
//------------------------------------------------------------------------------
// Buffer definitions.
//
// The logger will use SdFat's buffer plus BUFFER_BLOCK_COUNT-1 additional
// buffers.
//
#ifndef RAMEND
// Assume ARM. Use total of ten 512 byte buffers.
const uint8_t BUFFER_BLOCK_COUNT = 10;
//
#elif RAMEND < 0X8FF
#error Too little SRAM
//
#elif RAMEND < 0X10FF
// Use total of two 512 byte buffers.
const uint8_t BUFFER_BLOCK_COUNT = 2;
//
#elif RAMEND < 0X20FF
// Use total of four 512 byte buffers.
const uint8_t BUFFER_BLOCK_COUNT = 4;
//
#else // RAMEND
// Use total of 12 512 byte buffers.
const uint8_t BUFFER_BLOCK_COUNT = 12;
#endif // RAMEND
//==============================================================================
// End of configuration constants.
//==============================================================================
// Temporary log file. Will be deleted if a reset or power failure occurs.
#define TMP_FILE_NAME FILE_BASE_NAME "##.bin"
// Size of file base name.
const uint8_t BASE_NAME_SIZE = sizeof(FILE_BASE_NAME) - 1;
const uint8_t FILE_NAME_DIM = BASE_NAME_SIZE + 7;
char binName[FILE_NAME_DIM] = FILE_BASE_NAME "00.bin";
SdFat sd;
SdBaseFile binFile;
// Number of data records in a block.
const uint16_t DATA_DIM = (512 - 4)/sizeof(data_t);
//Compute fill so block size is 512 bytes. FILL_DIM may be zero.
const uint16_t FILL_DIM = 512 - 4 - DATA_DIM*sizeof(data_t);
struct block_t {
uint16_t count;
uint16_t overrun;
data_t data[DATA_DIM];
uint8_t fill[FILL_DIM];
};
//==============================================================================
// Error messages stored in flash.
#define error(msg) {sd.errorPrint(&Serial, F(msg));fatalBlink();}
//------------------------------------------------------------------------------
//
void fatalBlink() {
while (true) {
SysCall::yield();
if (ERROR_LED_PIN >= 0) {
digitalWrite(ERROR_LED_PIN, HIGH);
delay(200);
digitalWrite(ERROR_LED_PIN, LOW);
delay(200);
}
}
}
//------------------------------------------------------------------------------
// read data file and check for overruns
void checkOverrun() {
bool headerPrinted = false;
block_t block;
uint32_t bn = 0;
if (!binFile.isOpen()) {
Serial.println();
Serial.println(F("No current binary file"));
return;
}
binFile.rewind();
Serial.println();
Serial.print(F("FreeStack: "));
Serial.println(FreeStack());
Serial.println(F("Checking overrun errors - type any character to stop"));
while (binFile.read(&block, 512) == 512) {
if (block.count == 0) {
break;
}
if (block.overrun) {
if (!headerPrinted) {
Serial.println();
Serial.println(F("Overruns:"));
Serial.println(F("fileBlockNumber,sdBlockNumber,overrunCount"));
headerPrinted = true;
}
Serial.print(bn);
Serial.print(',');
Serial.print(binFile.firstBlock() + bn);
Serial.print(',');
Serial.println(block.overrun);
}
bn++;
}
if (!headerPrinted) {
Serial.println(F("No errors found"));
} else {
Serial.println(F("Done"));
}
}
//-----------------------------------------------------------------------------
// Convert binary file to csv file.
void binaryToCsv() {
uint8_t lastPct = 0;
block_t block;
uint32_t t0 = millis();
uint32_t syncCluster = 0;
SdFile csvFile;
char csvName[FILE_NAME_DIM];
if (!binFile.isOpen()) {
Serial.println();
Serial.println(F("No current binary file"));
return;
}
Serial.println();
Serial.print(F("FreeStack: "));
Serial.println(FreeStack());
// Create a new csvFile.
strcpy(csvName, binName);
strcpy(&csvName[BASE_NAME_SIZE + 3], "csv");
if (!csvFile.open(csvName, O_WRONLY | O_CREAT | O_TRUNC)) {
error("open csvFile failed");
}
binFile.rewind();
Serial.print(F("Writing: "));
Serial.print(csvName);
Serial.println(F(" - type any character to stop"));
printHeader(&csvFile);
uint32_t tPct = millis();
while (!Serial.available() && binFile.read(&block, 512) == 512) {
uint16_t i;
if (block.count == 0 || block.count > DATA_DIM) {
break;
}
if (block.overrun) {
csvFile.print(F("OVERRUN,"));
csvFile.println(block.overrun);
}
for (i = 0; i < block.count; i++) {
printData(&csvFile, &block.data[i]);
}
if (csvFile.curCluster() != syncCluster) {
csvFile.sync();
syncCluster = csvFile.curCluster();
}
if ((millis() - tPct) > 1000) {
uint8_t pct = binFile.curPosition()/(binFile.fileSize()/100);
if (pct != lastPct) {
tPct = millis();
lastPct = pct;
Serial.print(pct, DEC);
Serial.println('%');
}
}
if (Serial.available()) {
break;
}
}
csvFile.close();
Serial.print(F("Done: "));
Serial.print(0.001*(millis() - t0));
Serial.println(F(" Seconds"));
}
//-----------------------------------------------------------------------------
void createBinFile() {
// max number of blocks to erase per erase call
const uint32_t ERASE_SIZE = 262144L;
uint32_t bgnBlock, endBlock;
// Delete old tmp file.
if (sd.exists(TMP_FILE_NAME)) {
Serial.println(F("Deleting tmp file " TMP_FILE_NAME));
if (!sd.remove(TMP_FILE_NAME)) {
error("Can't remove tmp file");
}
}
// Create new file.
Serial.println(F("\nCreating new file"));
binFile.close();
if (!binFile.createContiguous(TMP_FILE_NAME, 512 * FILE_BLOCK_COUNT)) {
error("createContiguous failed");
}
// Get the address of the file on the SD.
if (!binFile.contiguousRange(&bgnBlock, &endBlock)) {
error("contiguousRange failed");
}
// Flash erase all data in the file.
Serial.println(F("Erasing all data"));
uint32_t bgnErase = bgnBlock;
uint32_t endErase;
while (bgnErase < endBlock) {
endErase = bgnErase + ERASE_SIZE;
if (endErase > endBlock) {
endErase = endBlock;
}
if (!sd.card()->erase(bgnErase, endErase)) {
error("erase failed");
}
bgnErase = endErase + 1;
}
}
//------------------------------------------------------------------------------
// dump data file to Serial
void dumpData() {
block_t block;
if (!binFile.isOpen()) {
Serial.println();
Serial.println(F("No current binary file"));
return;
}
binFile.rewind();
Serial.println();
Serial.println(F("Type any character to stop"));
delay(1000);
printHeader(&Serial);
while (!Serial.available() && binFile.read(&block , 512) == 512) {
if (block.count == 0) {
break;
}
if (block.overrun) {
Serial.print(F("OVERRUN,"));
Serial.println(block.overrun);
}
for (uint16_t i = 0; i < block.count; i++) {
printData(&Serial, &block.data[i]);
}
}
Serial.println(F("Done"));
}
//------------------------------------------------------------------------------
// log data
void logData() {
createBinFile();
recordBinFile();
renameBinFile();
}
//------------------------------------------------------------------------------
void openBinFile() {
char name[FILE_NAME_DIM];
strcpy(name, binName);
Serial.println(F("\nEnter two digit version"));
Serial.write(name, BASE_NAME_SIZE);
for (int i = 0; i < 2; i++) {
while (!Serial.available()) {
SysCall::yield();
}
char c = Serial.read();
Serial.write(c);
if (c < '0' || c > '9') {
Serial.println(F("\nInvalid digit"));
return;
}
name[BASE_NAME_SIZE + i] = c;
}
Serial.println(&name[BASE_NAME_SIZE+2]);
if (!sd.exists(name)) {
Serial.println(F("File does not exist"));
return;
}
binFile.close();
strcpy(binName, name);
if (!binFile.open(binName, O_RDONLY)) {
Serial.println(F("open failed"));
return;
}
Serial.println(F("File opened"));
}
//------------------------------------------------------------------------------
void recordBinFile() {
const uint8_t QUEUE_DIM = BUFFER_BLOCK_COUNT + 1;
// Index of last queue location.
const uint8_t QUEUE_LAST = QUEUE_DIM - 1;
// Allocate extra buffer space.
block_t block[BUFFER_BLOCK_COUNT - 1];
block_t* curBlock = 0;
block_t* emptyStack[BUFFER_BLOCK_COUNT];
uint8_t emptyTop;
uint8_t minTop;
block_t* fullQueue[QUEUE_DIM];
uint8_t fullHead = 0;
uint8_t fullTail = 0;
// Use SdFat's internal buffer.
emptyStack[0] = (block_t*)sd.vol()->cacheClear();
if (emptyStack[0] == 0) {
error("cacheClear failed");
}
// Put rest of buffers on the empty stack.
for (int i = 1; i < BUFFER_BLOCK_COUNT; i++) {
emptyStack[i] = &block[i - 1];
}
emptyTop = BUFFER_BLOCK_COUNT;
minTop = BUFFER_BLOCK_COUNT;
// Start a multiple block write.
if (!sd.card()->writeStart(binFile.firstBlock())) {
error("writeStart failed");
}
Serial.print(F("FreeStack: "));
Serial.println(FreeStack());
Serial.println(F("Logging - type any character to stop"));
bool closeFile = false;
uint32_t bn = 0;
uint32_t maxLatency = 0;
uint32_t overrun = 0;
uint32_t overrunTotal = 0;
uint32_t logTime = micros();
while(1) {
// Time for next data record.
logTime += LOG_INTERVAL_USEC;
if (Serial.available()) {
closeFile = true;
}
if (closeFile) {
if (curBlock != 0) {
// Put buffer in full queue.
fullQueue[fullHead] = curBlock;
fullHead = fullHead < QUEUE_LAST ? fullHead + 1 : 0;
curBlock = 0;
}
} else {
if (curBlock == 0 && emptyTop != 0) {
curBlock = emptyStack[--emptyTop];
if (emptyTop < minTop) {
minTop = emptyTop;
}
curBlock->count = 0;
curBlock->overrun = overrun;
overrun = 0;
}
if ((int32_t)(logTime - micros()) < 0) {
error("Rate too fast");
}
int32_t delta;
do {
delta = micros() - logTime;
} while (delta < 0);
if (curBlock == 0) {
overrun++;
overrunTotal++;
if (ERROR_LED_PIN >= 0) {
digitalWrite(ERROR_LED_PIN, HIGH);
}
#if ABORT_ON_OVERRUN
Serial.println(F("Overrun abort"));
break;
#endif // ABORT_ON_OVERRUN
} else {
#if USE_SHARED_SPI
sd.card()->spiStop();
#endif // USE_SHARED_SPI
acquireData(&curBlock->data[curBlock->count++]);
#if USE_SHARED_SPI
sd.card()->spiStart();
#endif // USE_SHARED_SPI
if (curBlock->count == DATA_DIM) {
fullQueue[fullHead] = curBlock;
fullHead = fullHead < QUEUE_LAST ? fullHead + 1 : 0;
curBlock = 0;
}
}
}
if (fullHead == fullTail) {
// Exit loop if done.
if (closeFile) {
break;
}
} else if (!sd.card()->isBusy()) {
// Get address of block to write.
block_t* pBlock = fullQueue[fullTail];
fullTail = fullTail < QUEUE_LAST ? fullTail + 1 : 0;
// Write block to SD.
uint32_t usec = micros();
if (!sd.card()->writeData((uint8_t*)pBlock)) {
error("write data failed");
}
usec = micros() - usec;
if (usec > maxLatency) {
maxLatency = usec;
}
// Move block to empty queue.
emptyStack[emptyTop++] = pBlock;
bn++;
if (bn == FILE_BLOCK_COUNT) {
// File full so stop
break;
}
}
}
if (!sd.card()->writeStop()) {
error("writeStop failed");
}
Serial.print(F("Min Free buffers: "));
Serial.println(minTop);
Serial.print(F("Max block write usec: "));
Serial.println(maxLatency);
Serial.print(F("Overruns: "));
Serial.println(overrunTotal);
// Truncate file if recording stopped early.
if (bn != FILE_BLOCK_COUNT) {
Serial.println(F("Truncating file"));
if (!binFile.truncate(512L * bn)) {
error("Can't truncate file");
}
}
}
//------------------------------------------------------------------------------
void recoverTmpFile() {
uint16_t count;
if (!binFile.open(TMP_FILE_NAME, O_RDWR)) {
return;
}
if (binFile.read(&count, 2) != 2 || count != DATA_DIM) {
error("Please delete existing " TMP_FILE_NAME);
}
Serial.println(F("\nRecovering data in tmp file " TMP_FILE_NAME));
uint32_t bgnBlock = 0;
uint32_t endBlock = binFile.fileSize()/512 - 1;
// find last used block.
while (bgnBlock < endBlock) {
uint32_t midBlock = (bgnBlock + endBlock + 1)/2;
binFile.seekSet(512*midBlock);
if (binFile.read(&count, 2) != 2) error("read");
if (count == 0 || count > DATA_DIM) {
endBlock = midBlock - 1;
} else {
bgnBlock = midBlock;
}
}
// truncate after last used block.
if (!binFile.truncate(512*(bgnBlock + 1))) {
error("Truncate " TMP_FILE_NAME " failed");
}
renameBinFile();
}
//-----------------------------------------------------------------------------
void renameBinFile() {
while (sd.exists(binName)) {
if (binName[BASE_NAME_SIZE + 1] != '9') {
binName[BASE_NAME_SIZE + 1]++;
} else {
binName[BASE_NAME_SIZE + 1] = '0';
if (binName[BASE_NAME_SIZE] == '9') {
error("Can't create file name");
}
binName[BASE_NAME_SIZE]++;
}
}
if (!binFile.rename(binName)) {
error("Can't rename file");
}
Serial.print(F("File renamed: "));
Serial.println(binName);
Serial.print(F("File size: "));
Serial.print(binFile.fileSize()/512);
Serial.println(F(" blocks"));
}
//------------------------------------------------------------------------------
void testSensor() {
const uint32_t interval = 200000;
int32_t diff;
data_t data;
Serial.println(F("\nTesting - type any character to stop\n"));
// Wait for Serial Idle.
delay(1000);
printHeader(&Serial);
uint32_t m = micros();
while (!Serial.available()) {
m += interval;
do {
diff = m - micros();
} while (diff > 0);
acquireData(&data);
printData(&Serial, &data);
}
}
//------------------------------------------------------------------------------
void setup(void) {
if (ERROR_LED_PIN >= 0) {
pinMode(ERROR_LED_PIN, OUTPUT);
}
Serial.begin(9600);
// Wait for USB Serial
while (!Serial) {
SysCall::yield();
}
Serial.print(F("\nFreeStack: "));
Serial.println(FreeStack());
Serial.print(F("Records/block: "));
Serial.println(DATA_DIM);
if (sizeof(block_t) != 512) {
error("Invalid block size");
}
// Allow userSetup access to SPI bus.
pinMode(SD_CS_PIN, OUTPUT);
digitalWrite(SD_CS_PIN, HIGH);
// Setup sensors.
userSetup();
// Initialize at the highest speed supported by the board that is
// not over 50 MHz. Try a lower speed if SPI errors occur.
if (!sd.begin(SD_CS_PIN, SD_SCK_MHZ(50))) {
sd.initErrorPrint(&Serial);
fatalBlink();
}
// recover existing tmp file.
if (sd.exists(TMP_FILE_NAME)) {
Serial.println(F("\nType 'Y' to recover existing tmp file " TMP_FILE_NAME));
while (!Serial.available()) {
SysCall::yield();
}
if (Serial.read() == 'Y') {
recoverTmpFile();
} else {
error("'Y' not typed, please manually delete " TMP_FILE_NAME);
}
}
}
//------------------------------------------------------------------------------
void loop(void) {
// Read any Serial data.
do {
delay(10);
} while (Serial.available() && Serial.read() >= 0);
Serial.println();
Serial.println(F("type:"));
Serial.println(F("b - open existing bin file"));
Serial.println(F("c - convert file to csv"));
Serial.println(F("d - dump data to Serial"));
Serial.println(F("e - overrun error details"));
Serial.println(F("l - list files"));
Serial.println(F("r - record data"));
Serial.println(F("t - test without logging"));
while(!Serial.available()) {
SysCall::yield();
}
#if WDT_YIELD_TIME_MICROS
Serial.println(F("LowLatencyLogger can not run with watchdog timer"));
SysCall::halt();
#endif
char c = tolower(Serial.read());
// Discard extra Serial data.
do {
delay(10);
} while (Serial.available() && Serial.read() >= 0);
if (ERROR_LED_PIN >= 0) {
digitalWrite(ERROR_LED_PIN, LOW);
}
if (c == 'b') {
openBinFile();
} else if (c == 'c') {
binaryToCsv();
} else if (c == 'd') {
dumpData();
} else if (c == 'e') {
checkOverrun();
} else if (c == 'l') {
Serial.println(F("\nls:"));
sd.ls(&Serial, LS_SIZE);
} else if (c == 'r') {
logData();
} else if (c == 't') {
testSensor();
} else {
Serial.println(F("Invalid entry"));
}
}