Data Logging Data structure

File content as of revision 11:bf816d33be80:

/* DataLog.h - Data logger for logging enviornmental
  data to EPROM, SRAM or FRAM chip without a file system
  inteface.  Supports multiple and evolving records types
  and trnsport back to serial 
  
  See data-log-text.txt for detailed design and layout notes
  See: xj-data-log-test-and-example.c for example use.
       https://developer.mbed.org/users/joeata2wh/code/xj-data-log-test-and-example/wiki/Homepage

  By Joseph Ellsworth CTO of A2WH
  Take a look at A2WH.com Producing Water from Air using Solar Energy
  March-2016 License: https://developer.mbed.org/handbook/MIT-Licence 
  Please contact us http://a2wh.com for help with custom design projects.
  
  Before you complain about not using proper C++ classes, I intend to 
  port the entire A2WH project to PSoc where I may or may not be able to
  use the full set of C++ features.  I am using techniques that should be
  easier to port to a ANSI C enviornment.  You may think this is a wierd
  decision but the PSoC enviornments gives me transparant support for 
  differential ADC and very low power analog comparators that remain active
  when CPU is in deep sleep and which can wake the CPU up from deep sleep
  which makes very low power designs easier.  mBed is still weak for this
  kind of advanced peripherial support. 

*/
#ifndef DataLog_H
#define DataLog_H
#include "mbed.h"
#include "multi-serial-command-listener.h"

#include "W25Q80BV.h"  // Note:  We are using this library because we started with it but
                       //  the actual chip we are using is a 2 MBit FRAM MB85RS2MTPH-G-JNE
                       //  also tested with SRAM 23LCV1024-I/P  Prefer SRAM or FRAM because
                       //  we made a simplifying assumption that we could write next log 
                       //  address to the same position over and over.  Without wear leveling
                       //  this could rapidly wear out a e-prom chip.  Could have written 
                       //  this to the clock chip which uses SRAM but FRAM has such high write
                       //  durability that we don't have to. 
                       
#define DataLogChipType W25Q80BV

// TODO: Add the SDIO link here to copy data from dlog chip to 
//   SD card when available. 


const long dlChipMaxAddr = 250000; // 2 mBit 2000000 / 8
#define dlChipFullErr -2
#define dlAddressTooLarge -3
#define dlMaxOperationSize -4;
// Chip DLog Chip Memory Layout
const long dlAddrInitByte  = 15000; // data before this is assumed to be used for system config variables
const char dlInitByteValue = 214;
const int dlMaxReadWriteSize = 32000; // limit imposed by streaming interface for the chip
const long dlAddrNextWritePos = dlAddrInitByte + 1;
const long dlNextWritePosSize = 4;
const long dlAddrCurrYDay = dlAddrNextWritePos + dlNextWritePosSize;
const long dlCurrYDaySize = 2;
const long dlAddrHeaders =   dlAddrCurrYDay + dlCurrYDaySize ;
const long dlHeadersLen  =   256;
const long dlAddrDateIndex  =   dlAddrHeaders + dlHeadersLen + 1;
const long dlDateIndexLen =   1000;
const long dlAddrFirstLogEntry=  dlAddrDateIndex + dlDateIndexLen + 1;
const char dlEmpty[] = {0,0,0,0,0,0,0};
const long dlMaxLogSize = dlChipMaxAddr - dlAddrFirstLogEntry;
#define MIN(X,Y) X <? Y
#define MAX(X,Y) X >? Y

struct DLOG {
   DataLogChipType *chip;
   long nextWritePos;
   int  currYDay; // tm_yday from gmtime only log date date when date changes
   char *buff;
   int  buffLen;
   Serial *sio;
   struct SCMD *cmdProc;
   
};

//-- Utility functions for exchanging data with Memory Chip
void ecWriteByte(DataLogChipType *chip, long addr, char tin){
    chip->writeStream(addr, (char *) &tin, sizeof(char));   
}

void ecWriteInt(DataLogChipType *chip, long addr, int tin){
    chip->writeStream(addr, (char *) &tin, sizeof(int));   
}

void ecWriteLong(DataLogChipType *chip, long addr, long tin){
    chip->writeStream(addr, (char *) &tin, sizeof(long));  
}

void ecWriteFloat(DataLogChipType *chip, long addr, float tin){
    chip->writeStream(addr, (char *) &tin, sizeof(float));  
}

void ecWriteStr(DataLogChipType *chip, long addr, char *tin){
    int slen = strlen(tin);
    chip->writeStream(addr, tin, slen+1);   
}

char ecReadByte(DataLogChipType *chip, long addr){
    char tval;
    chip->readStream(addr, (char *) &tval, sizeof(char));   
    return tval;
}

int ecReadInt(DataLogChipType *chip, long addr, int tin){
    int tval;
    chip->readStream(addr, (char *) &tval, sizeof(int));   
    return tval;
}

long ecReadLong(DataLogChipType *chip, long addr, long tin){
    long tval;
    chip->readStream(addr, (char *) &tval, sizeof(long));  
    return tval;
}

float ecReadFloat(DataLogChipType *chip, long addr, float tin){
    float tval;
    chip->readStream(addr, (char *) &tval, sizeof(float));  
    return tval;
}


// -- BEGIN DATA LOG --

void dlCommandProc(char *cmd,  void *dwrk);

long dlLen(struct DLOG *wrk) {
    return wrk->nextWritePos - dlAddrFirstLogEntry;
    }
// save the current nextWritePos to the chip so we hav it
// just in case of a reboot
void dlSaveNextWritePos(struct DLOG *wrk) {
   wrk->chip->writeStream(dlAddrNextWritePos,(char *) &wrk->nextWritePos,dlNextWritePosSize);   // write next write postion
}

long dlReadNextWritePos(struct DLOG *wrk) {
  wrk->chip->readStream(dlAddrNextWritePos, (char *) &wrk->nextWritePos, dlNextWritePosSize);
  //printf("dlReadNextWritePos wrk->nextWritePos=%ld\n\r",wrk->nextWritePos);
  return wrk->nextWritePos;
}

int dlReadCurrYDay(struct DLOG *wrk) {
  wrk->chip->readStream(dlAddrCurrYDay, (char *) &wrk->currYDay, dlCurrYDaySize);
  //printf("dlReadCurrYDay wrk->currYDay=%d\r\n", wrk->currYDay);
  return wrk->currYDay;  
}

void dlUpdateCurrDate(struct DLOG *wrk, int newYDay) {
  wrk->currYDay = newYDay;
  wrk->chip->writeStream(dlAddrCurrYDay,(char *) &wrk->currYDay, dlCurrYDaySize);   // write next write postion
}

// Erase Log from Chip but do not touch 
// data on chip outside of log space.
void dlEraseLog(struct DLOG *wrk) {
   //printf("dlEraseLogStart\r\n");
   wrk->nextWritePos =  dlAddrFirstLogEntry;
   wrk->currYDay = -99;
   memset(wrk->buff,0,wrk->buffLen);
   wrk->chip->writeStream(dlAddrInitByte, (char *) &dlInitByteValue,1);  // write init byte
   wrk->chip->writeStream(dlAddrNextWritePos,(char *) &wrk->nextWritePos,dlNextWritePosSize);   // reset the next write postion            
   wrk->chip->writeStream(dlAddrCurrYDay, (char *) &wrk->currYDay, dlCurrYDaySize); // Reset the currYDay
   wrk->chip->writeStream(dlAddrHeaders,wrk->buff,MIN(wrk->buffLen,dlHeadersLen));   // nulls over the header region
   wrk->chip->writeStream(dlAddrDateIndex,wrk->buff,MIN(wrk->buffLen,dlDateIndexLen));   // nulls over the header region   
   wrk->chip->writeStream(wrk->nextWritePos,wrk->buff,wrk->buffLen);   // nulls first of the log
   //printf("dlEraseLogDone\r\n");
}

// New data log chip detected write data to initialize it.
long dlInitializeChip(struct DLOG *wrk) {
   dlEraseLog(wrk);
   return wrk->nextWritePos;
}



/* read a initialization byte from chip.  If the byte
doesn't contain the expected value then write one
and assume that we are starting our log ad the beginning */
long dlCheckChipInit(struct DLOG *wrk){
  wrk->buff[0] = 0; 
  wrk->chip->readStream(dlAddrInitByte, wrk->buff, 1);
  if (wrk->buff[0] != dlInitByteValue) 
  {
    //printf("Found empty chip running init");
    return dlInitializeChip(wrk);      
    }
  else {
      //printf("Found existing log\r\n");
      dlReadCurrYDay(wrk);
      return dlReadNextWritePos(wrk);
    }
}



// make and instance of our dlog structure fill it in
// in and load any current data such as next write postion
// already loaded in the chip. 
struct DLOG *dlMake(DataLogChipType *dataLogMem, char *buff, short buffLen, Serial *sio) {  
  struct DLOG *tout = (struct DLOG *) malloc(sizeof(struct DLOG));
  tout->chip  = dataLogMem;
  tout->nextWritePos = dlAddrFirstLogEntry;
  tout->buff = buff;
  tout->buffLen = buffLen;
  tout->sio = sio;
  tout->cmdProc = scMake(sio, dlCommandProc, tout)  ;
  dlCheckChipInit(tout);
  return tout;
}


// writes log stream entry to chip and updates the next write 
// position. Also adds a null terminator to data on chip
// log entries should not contain null characters because we
// eventually plant to delay flush of nextWritePos and use
// scan forware to find the end when a crash occurs. 
// returns -2 if the write request would go beyond chip size.
long dlWrite(struct DLOG *wrk, char *aStr) {
  int slen = strlen(aStr);
  if ((wrk->nextWritePos + slen) >= dlChipMaxAddr) {
      return dlChipFullErr;
      }
  wrk->chip->writeStream(wrk->nextWritePos, aStr,slen);         
  wrk->nextWritePos += slen;
  dlSaveNextWritePos(wrk); // WARNING THIS IS THE LINE THAT WILL KILL EPROM CHIPS 
                           // with over-write fatigue.
  // TODO: add err check read first and last bytes.
  // compare to what was written.
  return wrk->nextWritePos;
}
char dlLFEmpty[] = "\n\000";
//record log file entry with time stamp and header
long dlLog(struct DLOG *wrk, char *recType, char *str) {
  
  time_t seconds;
  time(&seconds);
  //printf("dlLog ctime(&seconds)=%s\r\n", ctime(&seconds));
  struct tm *ptm = localtime( &seconds );
  //printf("dlLog asctime=%s\r\n", asctime(ptm));
  //printf("dlLog ptm->tm_yday=%d\r\n", ptm->tm_yday );
  if (ptm->tm_yday != wrk->currYDay) {
     int year = 1900 + ptm->tm_year;
     int month= ptm->tm_mon + 1;
     
     dlUpdateCurrDate(wrk, ptm->tm_yday);
     sprintf(wrk->buff,"\n00:00:00 DATE\t%04d-%02d-%02d", year, month, ptm->tm_mday);
     dlWrite(wrk, wrk->buff);
     //printf("update date dateRec=%s\r\n", wrk->buff);
  }
  
  memset(wrk->buff,wrk->buffLen,0);
  sprintf(wrk->buff,"\n%02d:%02d:%02d %s\t", ptm->tm_hour, ptm->tm_min, ptm->tm_sec, recType);  
  dlWrite(wrk, wrk->buff);     
  //dlWrite(wrk, dlLFEmpty);  // add terminating lineFeed
  //printf ("recHead=%s\r\n", wrk->buff);
  return dlWrite(wrk, str);
}

// read a block of bytes from log starting at offset
// for len bytes placed in buffer.  If offset is > 
// log size then return dlAddressTooLarge if len would
// be greate than log size only return that available.
long dlRead(struct DLOG *wrk, char *buff, long offset, int len) {
   long addr = dlAddrFirstLogEntry + offset;
   if ((addr + len) >= dlChipMaxAddr)
      return dlAddressTooLarge;        
   wrk->chip->readStream(offset, buff, len);       
   return 1;
}

long dlReadSend(struct DLOG *wrk, Serial *dest, long offset, long len) {
   long addr = dlAddrFirstLogEntry + offset;
   long maxAddr = MIN(addr + len, dlChipMaxAddr); // no overflow past end of chip
   maxAddr = MIN(maxAddr, wrk->nextWritePos); // no overlow pas end of log
   int chunkSize = wrk->buffLen -1;
   long endAdd = MIN(addr + len, maxAddr);   
   //printf("\r\ndlReadSend addr=%ld offset=%ld len=%ld maxAddr=%ld chunkSize=%d endAdd=%ld nextWritePos=%ld\r\n", 
   //  addr, offset, len, maxAddr, chunkSize, endAdd, wrk->nextWritePos);
   if (addr >= maxAddr)  
     return dlAddressTooLarge;
   int chunkCnt=0;
   do {
      memset(wrk->buff, wrk->buffLen, 0);
      if (addr + chunkSize > endAdd)
        chunkSize = endAdd - addr;
      //printf("\r\n\tdlReadSend addr=%ld endAdd=%ld chunkSize=%ld chunkCnt=%d\r\n", addr, endAdd, chunkSize, chunkCnt);
      wrk->chip->readStream(addr, wrk->buff, chunkSize);   
      wrk->buff[chunkSize] = 0;
      dest->printf("%s", wrk->buff); 
      int tlen = strlen(wrk->buff);
      //printf("\r\n\tdlReadSend tlen=%d\r\n", tlen);
      addr += chunkSize;      
      wait(0.06);
      chunkCnt++;
   } while (addr < endAdd);       
   return 1;
 }  
 
long dlReadSendAll(struct DLOG *wrk, Serial *dest) {
   time_t sec = time(NULL);
   dest->printf("\r\n\r\nDataLog READ ALL %s logSize=%d\r\n", ctime(&sec),wrk->nextWritePos);
   long res = dlReadSend(wrk, dest, 0, wrk->nextWritePos);
   dest->printf("\r\n\r\nDataLog FINISHED SEND ALL res=%d\r\n\r\n", res);
   return res;
 }
 
long dlReadSendLast(struct DLOG *wrk, Serial *dest, long numSend) {
   time_t sec = time(NULL);
   dest->printf("\r\n\r\nDataLog Read Send Last %s numSend=%d \r\n", ctime(&sec), numSend);
   long logLen = dlLen(wrk);
   long calcLen = MIN(numSend, logLen);
   long startPos = logLen - calcLen;
   startPos = MAX(0, startPos);
   dest->printf("\r\n\r\nRead Send Last numSend=%d logLen=%d, calcLen=%d startPos=%d\r\n",
     numSend, logLen, calcLen, startPos);
   //printf("dlReadSendLast logLen=%ld startPos=%ld calcLen=%ld nextWritePos=%ld\r\n ", logLen, startPos,calcLen, wrk->nextWritePos);
   long res = dlReadSend(wrk, dest, startPos, calcLen);
   printf ("\r\n\r\nfinished sendlast res=%d\r\n", res);
   return res;
}


void dlHelp(struct DLOG *wrk) {
    Serial *pc = wrk->sio;
    pc->printf("\r\n\r\nDataLog Help \r\n");
    pc->printf("\r\nCOMMANDS\r\n");
    pc->printf("  readall= send entire contents of log\r\n");
    pc->printf("  readlast 999\r\n");
    pc->printf("     999 = number of bytes from tail of log to retrieve\r\n");
    pc->printf("  tread 333 444\r\n");    
    pc->printf("     333 = starting offset to start reading log\r\n");
    pc->printf("     444 = number of bytes to retrieve from log\r\n");
    pc->printf("  erase = erase log and start a new one\r\n");
    pc->printf("  help  = display this help\r\n");
}

// recieve callback from serial console 
// parse those commands and take approapriate
// actions like sending back part of the data
// erasing the log etc. 
void dlCommandProc(char *cmd,  void *dwrk) {
  if (dwrk == NULL) return;
  struct DLOG *wrk = (struct DLOG *) dwrk;
  char cmdPrefix[10];
  long part2 = 0;
  long part3 = 0;
  sscanf (cmd,"%s %ld %ld",cmdPrefix, &part2, &part3);
  wrk->sio->printf("cmd=%s cmdPref=%s part2=%ld part3=%ld\r\n", cmd, cmdPrefix, part2, part3);

  if (strcmp(cmdPrefix,"readall") == 0) {
    dlReadSendAll(wrk, wrk->sio);
  } 
  else if (strcmp(cmdPrefix, "readlast") == 0) {
    dlReadSendLast(wrk, wrk->sio, part2);
  }
  else if (strcmp(cmdPrefix, "read") == 0) {
    dlReadSend(wrk, wrk->sio, part2, part3);
  }
  else if (strcmp(cmdPrefix, "erase") == 0) {
    wrk->sio->printf("\r\n\r\nDataLog Erase \r\n");
    dlEraseLog(wrk);
  }
  else if (strcmp(cmdPrefix, "help") == 0) {    
    dlHelp(wrk);
  }
  else {
    wrk->sio->printf("ERR no matching command cmd=%s\r\n", cmd); // no command matched 
    dlHelp(wrk);
  }  
}

#endif