File content as of revision 0:e2daaf858e13:

/* BlinkTalk - Bob Stone June 2013
 * Hands-free control of a speech synthesis system by blinking.
 *
 * This project implements a proof of concept speech system actuated by a single input
 * - this could be a button, but for a more fun application I am going to use blinks as
 * detected by an EEG headset, with a row & column time-sweep keyboard, inspired by
 * (but not the same as) the speech system used by Prof Stephen Hawking, mainly because the
 * Emic2 voice board includes the DECTalk 'PerfectPaul' voice familiar to all as Hawking.
 *
 * In Hawking's actual system, an infrared sensor mounted on his glasses detects a movement
 * of his cheek muscle, selecting letters from a keyboard  sweeping rows and columns, with a
 * predictive text system handling word completion and suggested follow-on words.  In
 * our version we will detect a blink using a Neurosky Mindwave Mobile headset and use it to
 * stop and select a cursor sweeping a keyboard grid.
 *
 * Hardware:
 *      Neurosky Mindwave Mobile headset - sends serial data packets over BlueTooth
 *      BlueSMiRF Silver Mate - receives data over BlueTooth from headset and relays over serial to mbed
 *      Parallax / Grand Idea Studio Emic2 speech synthesis - speaks text sent over serial from the mbed
 *      MikroElektronika TFT Proto - 320x240 TFT display with HX8374 controller driven by SPI
 *      Sparkfun level shifter - translates Emic2 5V serial to 3.3V serial for mbed
 *
 * Connections:
 *  mbed        BlueSMiRF   LevelShift  Emic2   Speaker TFT-PROTO
 *   GND         GND         GND/GND     GND             GND
 *   VOUT(3.3V)  VCC         LV                          3.3V
 *   VU (5V)                 HV          5V
 *   p9 (TX)     RX-I
 *   p10(RX)     TX-O
 *   p11(MOSI)                                           SDI
 *   p12(MISO)                                           SDO
 *   p13(SCLK)                                           SCL
 *   p14                                                 CS
 *   p15                                                 RST
 *   p27(RX)                 LV:RXO
 *   p28(TX)                 LV:TXI
 *                           HV:RXI      SOUT
 *                           HV:TXO      SN
 *                                       SP+     +
 *                                       SP-     -
 *      
 * Borrows from http://developer.neurosky.com/docs/doku.php?id=mindwave_mobile_and_arduino
 * Display library from Peter Drescher: http://mbed.org/cookbook/SPI-driven-QVGA-TFT
 */
#include "mbed.h"
#include "SPI_TFT.h"
#include "Arial12x12.h"
#include "Arial28x28.h"
#include <string>

//Peripherals
Serial blueSmirf(p9, p10);                      //bluetooth comms (TX, RX)
Serial voice(p28,p27);                          //Emic2 text to speech voice synth
SPI_TFT screen(p11, p12, p13, p14, p15, "TFT"); //display

//control variables
int quality=0;
bool connected=false;
bool started=false;
int row=-1;
int col=-1;
Timer initialDelay;

//Keyboard - rows to display
#define ROWS 5
#define COLS 10
//keys or text to display
string keys[ROWS][COLS] = {
    {"1","2","3","4","5","6","7","8","9","0"},
    {"Q","W","E","R","T","Y","U","I","O","P"},
    {"A","S","D","F","G","H","J","K","L","!"},
    {"Z","X","C","V","B","N","M",",",".","?"},
    {"Space","\"SAY!\"","Delete"}
};
//positions to draw a line at the end of a column
int keysColPos[ROWS][COLS] = {
    {36,67,98,129,160,191,222,253,284,315},
    {36,67,98,129,160,191,222,253,284,315},
    {36,67,98,129,160,191,222,253,284,315},
    {36,67,98,129,160,191,222,253,284,315},
    {100,210,315}
};
//number of elements in each row
int keysRowSize[ROWS] = {10, 10, 10, 10, 3};

//text being typed
string typingBuffer("");

//*****************************
//User routines to process data
//*****************************

/** Say some text out loud
 *
 * @param text The text to say out loud
 */
void say(string text)
{
    voice.printf("S%s\n", text);//send command to speak to the Emic2
    while(!voice.readable())    //wait for Emic2 to return ':'
        ;                       //do nothing whilst it's still speaking
    voice.getc();               //pop the ':' response from the stream
}

/** Maps a value from one scale to another
 *
 * @param value Value we're trying to scale
 * @param min,max The range that value came from
 * @param newMin,newMax The new range we're scaling value into
 * @returns value mapped into new scale
 */
int map(int value, int min, int max, int newMin, int newMax)
{
    if (min==max)
        return newMax;
    else
        return newMin + (newMax-newMin) * (value-min) / (max-min);
}

/** Returns a 16-bit RGB565 colour from three 8-bit component values.
 *
 * @param red,green,blue primary colour channel values expressed as 0-255 each
 * @returns 16-bit RGB565 colour constructed as RRRRRGGGGGGBBBBB
 */
int RGBColour(int red, int green, int blue)
{
    //take most-significant parts of red, green and blue and bit-shift into RGB565 positions
    return ((red & 0xf8) << 8) | ((green & 0xfc) << 3) | ((blue & 0xf8) >> 3);
}

/** Returns a colour mapped on a gradient from one colour to another.
 *
 * @param value Value we're trying to pick a colour for
 * @param min,max Scale that value belongs in
 * @param minColour,maxColour start and end colours of the gradient we're choosing from (16-bit RGB565)
 * @returns colour that's as far along the gradient from minColour to maxColour as value is between min and max (16-bit RGB565)
 */
int getMappedColour(int value, int min, int max, int minColour, int maxColour)
{
    // TFT screen colours are 16-bit RGB565 i.e. RRRRRGGGGGGBBBBB
    int minRed = (minColour & 0xf800) >> 11; //bitmask for 5 bits red
    int maxRed = (maxColour & 0xf800) >> 11;
    int minGreen = (minColour & 0x7e0) >> 5; //bitmask for 6 bits green
    int maxGreen = (maxColour & 0x7e0) >> 5;
    int minBlue = minColour & 0x1f; // bitmask for 5 bits blue
    int maxBlue = maxColour & 0x1f;
    int valRed = map(value, min, max, minRed, maxRed);
    int valGreen = map(value, min, max, minGreen, maxGreen);
    int valBlue = map(value, min, max, minBlue, maxBlue);
    int valColour = ((valRed & 0x1F) << 11) | ((valGreen & 0x3F) << 5) | (valBlue & 0x1F);
    return valColour;
}

/** Displays a bar graph showing 'value' on a scale 'min' to 'max', where coords (x0,y0) are at 'min' and (x1,y1) are at 'max'.
 *
 * @param x0,y0 coordinates of the 'min' end of the bargraph
 * @param x1,y1 coordinates of the 'max' end of the bargraph
 * @param isHorizontal If true, bar graph will be drawn with horizontal bars
 * @param value Value of the bar, with bars drawn from min up to value, remaining 'backColour' from there to max
 * @param min,max Scale of the bar graph that value should be found within
 * @param minColour,maxColour colours at the min and max ends of the bar, drawn in a gradient between the two (16-bit RGB565)
 * @param backColour background colour of the bar graph (16-bit RGB565)
 */
void displayBarGraph(int x0, int y0, int x1, int y1, bool isHorizontal, int value, int min, int max, int minColour, int maxColour, int backColour)
{
    int valColour;
    if (isHorizontal) {
        if (x1>x0) {
            for (int i = x0; i < x1; i+=5) {
                if (map(i, x0, x1, min, max) > value)
                    valColour = backColour;
                else
                    valColour = getMappedColour(i, x0, x1, minColour, maxColour);
                screen.fillrect(i, y0, i+3, y1, valColour);
            }
        } else {
            for (int i = x1; i < x0; i+=5) {
                if (map(i, x0, x1, min, max) > value)
                    valColour = backColour;
                else
                    valColour = getMappedColour(i, x0, x1, minColour, maxColour);
                screen.fillrect(i-3, y0, i, y1, valColour);
            }
        }
    } else {
        if (y1>y0) {
            for (int i = y0; i < y1; i+=5) {
                if (map(i, y0, y1, min, max) > value)
                    valColour = backColour;
                else
                    valColour = getMappedColour(i, y0, y1, minColour, maxColour);
                screen.fillrect(x0, i, x1, i+3, valColour);
            }
        } else {
            for (int i = y1; i < y0; i+=5) {
                if (map(i, y0, y1, min, max) > value)
                    valColour = backColour;
                else
                    valColour = getMappedColour(i, y0, y1, minColour, maxColour);
                screen.fillrect(x0, i-3, x1, i, valColour);
            }
        }
    }
}

/** Draw a keyboard based on the supplied array of cells
 *
 * @param cells the cells to draw
 * @param rowHighlight off if -1, else highlights the numbered row
 * @param colHighlight off if -1, else highlights the cell in row/col
 */
void drawKeyboard(int rowSize[ROWS], string cells[ROWS][COLS], int colPos[ROWS][COLS], int rowHighlight, int colHighlight)
{
    int lineColour = RGBColour(0x20,0xFF,0xE0);
    screen.foreground(RGBColour(0xE0,0xC0,0x10));
    screen.set_font((unsigned char*) Arial28x28);
    for (int i=0; i<ROWS; i++) {
        int yPos=111+i*32;
        for (int j=0; j< rowSize[i]; j++) {
            if (j>0)
                screen.locate(colPos[i][j-1]+5,84+i*32);
            else
                screen.locate(10,84+i*32);
            screen.printf("%s",cells[i][j]);
            screen.line(colPos[i][j],yPos-31,colPos[i][j],yPos, lineColour);
        }
        screen.line(5, yPos, 315, yPos, lineColour);
    }
    screen.line(5, 79, 315, 79, lineColour);
    screen.line(5, 79, 5, 239, lineColour);
    if (rowHighlight >= 0) {
        if (colHighlight >= 0) { //highlight a cell
            if (colHighlight==0)
                screen.rect(5,79+rowHighlight*32,colPos[rowHighlight][0],111+rowHighlight*32,RGBColour(0xFF,0x40,0x40));
            else
                screen.rect(colPos[rowHighlight][colHighlight-1],79+rowHighlight*32,colPos[rowHighlight][colHighlight],111+rowHighlight*32,RGBColour(0xFF,0x40,0x40));
        } else { // highlight whole row
            screen.rect(5,79+rowHighlight*32,315,111+rowHighlight*32,RGBColour(0xFF,0x40,0x40));
        }
    }
}

void drawText()
{
    screen.rect(5,35,315,74,RGBColour(0x20,0xFF,0xE0));
    screen.locate(10,40);
    screen.foreground(RGBColour(0xE0,0xC0,0x10));
    screen.set_font((unsigned char*) Arial28x28);
    screen.printf("%s_  ", typingBuffer);
}

/** This will be called if you blink.
 */
void blinked(void)
{
    //draw blink indicator
    if (quality == 0) {
        screen.fillrect(313, 13, 317, 17, White);
    }
    //select row or cell
    if (col == -1)
        col=-2;
    else {
        string s = keys[row][col];
        if (s.compare("Space") == 0)
            s=" ";
        if (s.compare("Delete") == 0) {
            s="";
            if (typingBuffer.length() > 0) {
                typingBuffer = typingBuffer.substr(0, typingBuffer.length() -1);
            }
        } else if (s.compare("\"SAY!\"") == 0) {
            say(typingBuffer);
            screen.fillrect(5,35,315,74,Black);
            typingBuffer="";
        } else {
            typingBuffer.append(s);
        }
        row = -1;
        col = -1;
        drawText();
    }
}

/** This will be called when processed eSense data comes in, about once a second.
 *
 * @param poorQuality will be 0 if connections are good, 200 if connections are useless, and somewhere in between if connection dodgy.
 * @param attention processed percentage denoting focus and attention.  0 to 100
 * @param meditation processed percentage denoting calmness and serenity.  0 to 100
 * @param timeSinceLastPacket time since last packet processed, in milliseconds.
 */
void eSenseData(int poorQuality, int attention, int meditation, int timeSinceLastPacket)
{
    //quality indicator
    quality=poorQuality;
    if (poorQuality == 200)
        screen.fillrect(313, 3, 317, 7, Red);
    else if (poorQuality == 0)
        screen.fillrect(313, 3, 317, 7, Green);
    else
        screen.fillrect(313, 3, 317, 7, Yellow);

    //minimal eSense bars up at the top of the screen
    screen.set_font((unsigned char*) Arial12x12);
    if (attention > 0) {
        displayBarGraph(200, 5, 310, 15, true, attention, 0, 100, RGBColour(0x10,0x00,0x00), RGBColour(0xFF,0x00,0x00), 0x00);
        screen.locate(135, 6);
        screen.foreground(Red);
        screen.printf("Att: %d ",attention);
    }
    if (meditation > 0) {
        displayBarGraph(200, 18, 310, 28, true, meditation, 0, 100, RGBColour(0x00,0x10,0x00), RGBColour(0x00,0xFF,0x00), 0x00);
        screen.locate(128, 19);
        screen.foreground(Green);
        screen.printf("Med: %d ",meditation);
    }
    //clear blink indicator
    screen.fillrect(313, 13, 317, 17, Black);
    //Safe to start yet?
    if (initialDelay.read() == 0)
        initialDelay.start();
    else if (initialDelay.read() > 5) {
        started=true;
        initialDelay.stop();
    }
}

/** This will be called when processed meter reading data arrives, about once a second.
 * This is a breakdown of frequencies in the wave data into 8 named bands, these are:
 *   0: Delta        (0.5-2.75 Hz)
 *   1: Theta        (3.5-6.75 Hz)
 *   2: Low-Alpha    (7.5-9.25 Hz)
 *   3: High-Alpha   (10-11.75 Hz)
 *   4: Low-Beta     (13-16.75 Hz)
 *   5: High-Beta    (18-29.75 Hz)
 *   6: Low-Gamma    (31-39.75 Hz)
 *   7: High-Gamma   (41-49.75 Hz)
 *
 * @param meter array of meter data for different frequency bands
 * @param meterMin array of minimum recorded samples of each band
 * @param meterMax arrat if naximum recorded samples of each band
 */
void meterData(int meter[8], int meterMin[8], int meterMax[8])
{
    //first good signal?
    if (!connected) {
        connected=true;
        screen.fillrect(0,0,319,30,Black); //clear the Waiting to connect msg
        initialDelay.reset();
        initialDelay.start();
    }
    //minimal meter bars up at the top of the screen
    for (int j=0; j<8; j++) {
        displayBarGraph(5 + j * 13, 30, 16 + j * 13, 3, false, meter[j], meterMin[j], meterMax[j], RGBColour(0, j*2, 0x10), RGBColour(0, j*32, 0xFF), 0x00);
    }
    //Hijack this routine for menu movement
    if (started) {
        if (col==-1) {
            row++;
            if (row>=ROWS)
                row=0;
        } else if (col==-2) {
            col=0;
        } else {
            col++;
            if (col>=keysRowSize[row])
                col=-1;
        }
        drawKeyboard(keysRowSize, keys, keysColPos, row, col);
    }
}

/** This will be called when wave data arrives.
 * There will be a lot of these, 512 a second, so if you're planning to do anything
 * here, don't let it take long.  Best not to printf this out as it will just choke.
 *
 * param wave Raw wave data point
 */
void waveData(int wave)
{
}

//*****************
//End User routines
//*****************

//System routines to obtain and parse data

/** Simplify serial comms
 */
unsigned char ReadOneByte()
{
    int ByteRead;

    while(!blueSmirf.readable());
    ByteRead = blueSmirf.getc();

    return ByteRead;
}

/** Main loop, sets up and keeps listening for serial
 */
int main()
{
    //Video setup
    screen.claim(stdout);        // send stdout to the TFT display
    screen.background(Black);    // set background to black
    screen.foreground(White);    // set chars to white
    screen.cls();                // clear the screen
    screen.set_orientation(1);
    screen.set_font((unsigned char*) Arial12x12);
    screen.locate(5,5);
    screen.printf("Waiting to connect...");

    drawText();
    drawKeyboard(keysRowSize, keys, keysColPos, -1, -1);

    //Voice setup
    voice.baud(9600);
    voice.printf("\n");
    while (!voice.readable())
        ;
    wait(0.01);
    voice.getc();
    say("Welcome to Blink talk.");

    Timer t; //packet timer
    t.start();
    Timer blinkTimer; //used for detecting blinks
    int time;
    int generatedChecksum = 0;
    int checksum = 0;
    int payloadLength = 0;
    int payloadData[64] = {0};
    int poorQuality = 0;
    int attention = 0;
    int meditation = 0;
    int wave = 0;
    int meter[8] = {0};
    int meterMin[8];
    int meterMax[8];
    for (int j = 0; j < 8; j++) {
        meterMin[j]=99999999;
        meterMax[j]=-99999999;
    }
    bool eSensePacket = false;
    bool meterPacket = false;
    bool wavePacket = false;

    blueSmirf.baud(57600);
    blinkTimer.reset();

    while(1) {
        // Look for sync bytes
        if(ReadOneByte() == 170) {
            if(ReadOneByte() == 170) {
                //Synchronised to start of packet
                payloadLength = ReadOneByte();
                if(payloadLength > 169) //Payload length can not be greater than 169
                    return;

                generatedChecksum = 0;
                for(int i = 0; i < payloadLength; i++) {
                    payloadData[i] = ReadOneByte();            //Read payload into memory
                    generatedChecksum += payloadData[i];
                }

                checksum = ReadOneByte();                      //Read checksum byte from stream
                generatedChecksum = 255 - (generatedChecksum & 0xFF);   //Take one's compliment of generated checksum

                if(checksum == generatedChecksum) {
                    //Packet seems OK
                    poorQuality = 200;
                    attention = 0;
                    meditation = 0;
                    wave = 0;
                    for(int i = 0; i < payloadLength; i++) {    // Parse the payload
                        switch (payloadData[i]) {
                            case 2: //quality
                                i++;
                                poorQuality = payloadData[i];
                                eSensePacket = true;
                                break;
                            case 4: //attention
                                i++;
                                attention = payloadData[i];
                                eSensePacket = true;
                                break;
                            case 5: //meditation
                                i++;
                                meditation = payloadData[i];
                                eSensePacket = true;
                                break;
                            case 0x80: //wave
                                wave = payloadData[i+2] * 256 + payloadData[i+3];
                                //We also want to try to detect blinks via analysing wave data
                                time = blinkTimer.read_ms();
                                if (wave > 32767) wave -= 65535; //cope with negatives
                                if (wave>200 && time == 0) {
                                    blinkTimer.start();
                                } else if (wave<-90 && time > 10 && time < 350) {
                                    blinkTimer.stop();
                                    blinkTimer.reset();
                                    blinked();
                                } else if (time>500) {
                                    blinkTimer.stop();
                                    blinkTimer.reset();
                                }
                                i = i + 3;
                                wavePacket = true;
                                break;
                            case 0x83: //meter readings for different frequency bands
                                for (int j=0; j<8; j++) {
                                    //documentation is inconsistent about whether these values are big-endian or little-endian,
                                    //and claims both in different places.  But wave data is big-endian so assuming that here.
                                    meter[j] = payloadData[i+j*3+2]*65536 + payloadData[i+j*3+3]*256 + payloadData[i+j*3+4];
                                    if (quality==0) {
                                        if (meter[j]<meterMin[j])
                                            meterMin[j]=meter[j];
                                        if (meter[j]>meterMax[j])
                                            meterMax[j]=meter[j];
                                    }
                                }
                                meterPacket = true;
                                i = i + 25;
                                break;
                            default:
                                break;
                        } // switch
                    } // for loop

                    //Call routines to process data
                    if(eSensePacket) {
                        eSenseData(poorQuality, attention, meditation, t.read_ms());
                        eSensePacket = false;
                    }
                    if (meterPacket) {
                        meterData(meter, meterMin, meterMax);
                        t.reset();
                        meterPacket=false;
                    }
                    if (wavePacket) {
                        waveData(wave);
                        wavePacket=false;
                    }
                } else {
                    // Checksum Error
                }  // end if else for checksum
            } // end if read 0xAA byte
        } // end if read 0xAA byte
    } //end while
}