Self Organizing Map – EEG AND MACHINE LEARNING IN OPENFRAMEWORKS

Posted on

Musical output from using self organizing map algorithm with data from a EEG Headset in realtime. Very happy and I can take the headset off and the program continues to generate the composition.

#pragma once

#include "ofMain.h"
#include "ofxAssignment.h"
#include "ofxThinkgear.h"
#include "ofxSelfOrganizingMap.h"
#include "ofxMidi.h"

class ofApp : public ofBaseApp{

	public:
		void setup();
		void update();
		void draw();

		void keyPressed(int key);
		void keyReleased(int key);
		void mouseMoved(int x, int y );
		void mouseDragged(int x, int y, int button);
		void mousePressed(int x, int y, int button);
		void mouseReleased(int x, int y, int button);
		void mouseEntered(int x, int y);
		void mouseExited(int x, int y);
		void windowResized(int w, int h);
		void dragEvent(ofDragInfo dragInfo);
		void gotMessage(ofMessage msg);
    
    void onThinkgearError(ofMessage& err);
    void onThinkgearReady(ofxThinkgearEventArgs& args);
    void onThinkgearRaw(ofxThinkgearEventArgs& args);
    void onThinkgearPower(ofxThinkgearEventArgs& args);
    void onThinkgearPoorSignal(ofxThinkgearEventArgs& args);
    void onThinkgearBlinkStrength(ofxThinkgearEventArgs& args);
    void onThinkgearAttention(ofxThinkgearEventArgs& args);
    void onThinkgearMeditation(ofxThinkgearEventArgs& args);
    void onThinkgearEeg(ofxThinkgearEventArgs& args);
    void onThinkgearConnecting(ofxThinkgearEventArgs& args);
    
    int delta;
    int theta;
    int lowalpha;
    int highalpha;
    int lowbeta;
    int highbeta;
    int lowgamma;
    int midgamma;
    int eegAttention;
    int eegMeditation;
    int avg;
    ofEasyCam cam;
    ofxAssignment solver;
    vector<ofVec2f> initial, grid;
    
    ofxSelfOrganizingMap som;
    ofImage somImg, sourceImg, sourceImgThumbs[5];
    string sourcePaths[5];
    ofImage img;
    ofMesh mesh;
    ofShader shader;
    ofPlanePrimitive plane;
    ofxMidiOut midiOut;
    
private:
    
    ofxThinkgear tg;
    ofxThinkgearEventArgs data;
    
};

#include "ofApp.h"
#include "ofxAssignment.h"
//--------------------------------------------------------------
void ofApp::setup(){
    
    tg.addEventListener(this);

    // In this example, we are making an SOM from an image file.  We are
    // using the image's RGB values as the features, and will be creating
    // a 2D map of the color distribution
    for (int i = 0; i < 5; i++) {
        sourcePaths[i] = "flower" + ofToString(i+1) + ofToString(".jpg");
        sourceImgThumbs[i].loadImage(sourcePaths[i]);
    }
    sourceImg.loadImage(sourcePaths[0]);
    sourceImg.allocate(100, 100, OF_IMAGE_COLOR);
    somImg.allocate(256, 256, OF_IMAGE_COLOR);
    
    // Before setting up the SOM, you must tell it the size of the
    // feature vector and the minimum and maximum values of each element
    double minInstance[3] = { 0, 0, 0 };
    double maxInstance[3] = { 255, 255, 255 };
    som.setFeaturesRange(3, minInstance, maxInstance);
    
    // Optional step: the initial learning rate is 0.1 by default, and the
    // number of iterations is 3000 by default.  In most cases, you
    // won't need to change this.
    // The initial learning rate controls how aggressive the map
    // update process is and the number of iterations is self-explanatory.
    // By setting the former higher and the latter lower, your map will
    // converge faster, but it may not converge on the most optimal
    // solution.  A lower learning rate with more iterations will usually
    // create a smoother map.
    // If you are seeing fuzziness or noisiness in any region of the map,
    // it means that you should increase either the rate or number of iterations
    // because the SOM did not have enough time to smooth out the noise.
    som.setInitialLearningRate(0.1);
    som.setNumIterations(3000000);
    
    // After you are with options, you initialize the SOM with setup
    som.setup();
    shader.load("shadersGL3/shader");    //float planeScale = 0.75;
    //int planeWidth = ofGetWidth() * planeScale;
    //int planeHeight = ofGetHeight() * planeScale;
    //int planeGridSize = 20;
    //int planeColumns = planeWidth / planeGridSize;
    //int planeRows = planeHeight / planeGridSize;
    
    //plane.set(planeWidth, planeHeight, planeColumns, planeRows, OF_PRIMITIVE_TRIANGLES);
    midiOut.listPorts();
    midiOut.openPort(1);
}

//--------------------------------------------------------------
void ofApp::update(){
    tg.update();
    
    float EEG_LOW_ALPHA_FLOAT = ofMap(EEG_LOW_ALPHA, 0, 1000, .01, .99);
    float EEG_DELTA_FLOAT = ofMap(EEG_DELTA, 0, 1000, .01, .99);
    
    sourceImg.setColor(EEG_LOW_ALPHA_FLOAT*100, EEG_DELTA_FLOAT*100, ofColor(float(EEG_LOW_ALPHA),float(EEG_DELTA),float(EEG_THETA)));
    sourceImg.update();
    // To train the SOM, we have to randomly sample instances from the
    // data we are modeling, in this case an RGB image.
    // The updateMap(instance) method will run exactly numIterations times,
    // after which the method stops affecting the map.
    
    // randomly sample a color from the source image and update map
    ofColor samp = sourceImg.getColor( ofRandom(sourceImg.getWidth()), ofRandom(sourceImg.getHeight()));
    double instance[3] = { static_cast<double>(samp.r), static_cast<double>(samp.g), static_cast<double>(samp.b) };
    som.updateMap(instance);
    midiOut.sendNoteOn(1, instance[0], instance[0]);
    midiOut.sendNoteOff(1, instance[0], instance[0]);
    midiOut.sendNoteOn(2, instance[1], instance[1]);
    midiOut.sendNoteOff(2, instance[1], instance[1]);
    midiOut.sendNoteOn(3, instance[2], instance[2]);
    midiOut.sendNoteOff(3, instance[2], instance[2]);
    // At any time, the map's value vector at a given point can be accessed
    // with the getMapAt method. In this example, we set somImg's pixels to those
    // vectors to see how the colors are distributed on the map.
    for (int i = 0; i < 256; i++) {

        for (int j = 0; j < 256; j++) {
            double * c = som.getMapAt(i,j);
            ofColor col(c[0], c[1], c[2]);
            somImg.setColor(i, j, col);

        }
    }

    somImg.update();
    
}

//--------------------------------------------------------------
void ofApp::draw(){
    
    ofBackground(0);
    ofDrawBitmapString("Source image", 30, 20);

    sourceImg.draw(0, 0, ofGetWidth(), ofGetHeight());

    somImg.draw(0, 0, ofGetWidth(), ofGetHeight());
    //img.draw(20, 30, 480, 480);
    ofDrawBitmapString("Self-organizing map: Iteration " + ofToString(som.getCurrentIteration()) + "/" + ofToString(som.getNumIterations()), 530, 20);
    // draw thumbnails
    //ofDrawBitmapString("Click one of the thumbnails or drag another image file into the application to change the source", 120, 540);
    for (int i = 0; i < 5; i++)
      //  sourceImgThumbs[i].draw(25 + 195*i, 550, 185, 185);
    //mesh.draw();
    ofSetColor(255);
    float percentX = mouseX / (float)ofGetWidth();
    percentX = ofClamp(percentX, 0, 1);
    
    // the mouse/touch X position changes the color of the plane.
    // please have a look inside the frag shader,
    // we are using the globalColor value that OF passes into the shader everytime you call ofSetColor().
    ofColor colorLeft = ofColor::magenta;
    ofColor colorRight = ofColor::cyan;
    ofColor colorMix = colorLeft.getLerped(float(eegAttention), float(eegMeditation));
    ofSetColor(colorMix);
    
    shader.begin(); // start shading!
    
    // a lot of the time you have to pass in variables into the shader.
    // in this case we need to pass it the elapsed time for the sine wave animation.
    shader.setUniform1f("time", ofGetElapsedTimef());
    
    // translate plane into center screen.
    float tx = ofGetWidth() / 2;
    float ty = ofGetHeight() / 2;
    ofTranslate(tx, ty);
    
    // the mouse/touch Y position changes the rotation of the plane.
    float percentY = eegMeditation / (float)ofGetHeight();
    float rotation = ofMap(percentY, 0, 1, -60, 60, true) + 60;
    ofRotate(rotation, 1, 0, 0);
    
    //plane.drawWireframe();
    
    shader.end();
}

//--------------------------------------------------------------
void ofApp::keyPressed(int key){

}

//--------------------------------------------------------------
void ofApp::keyReleased(int key){

}

//--------------------------------------------------------------
void ofApp::mouseMoved(int x, int y ){

}

//--------------------------------------------------------------
void ofApp::mouseDragged(int x, int y, int button){

}

//--------------------------------------------------------------
void ofApp::mousePressed(int x, int y, int button){
    som.setup();
}

//--------------------------------------------------------------
void ofApp::mouseReleased(int x, int y, int button){

}

//--------------------------------------------------------------
void ofApp::mouseEntered(int x, int y){

}

//--------------------------------------------------------------
void ofApp::mouseExited(int x, int y){

}

//--------------------------------------------------------------
void ofApp::windowResized(int w, int h){

}

//--------------------------------------------------------------
void ofApp::gotMessage(ofMessage msg){

}

//--------------------------------------------------------------
void ofApp::dragEvent(ofDragInfo info){
    if (info.files.size()>0) {
        //sourceImg.loadImage(info.files[0]);
        //som.setup();
    }
}

void ofApp::onThinkgearReady(ofxThinkgearEventArgs& args){
    cout << "onReady" << endl;
}

void ofApp::onThinkgearError(ofMessage& err){
    cout << "onError " << err.message << endl;
}

void ofApp::onThinkgearRaw(ofxThinkgearEventArgs& args){
    cout << "delta: " << EEG_DELTA << endl;
    //power.value = EEG_DELTA;
}

void ofApp::onThinkgearPower(ofxThinkgearEventArgs& args){
    // power.value = args.power;
}

void ofApp::onThinkgearPoorSignal(ofxThinkgearEventArgs& args){
    //poorsignal.value = args.poorSignal;
}

void ofApp::onThinkgearBlinkStrength(ofxThinkgearEventArgs& args){
    //blink.value = args.blinkStrength;
    cout << "BLINK: " << args.blinkStrength << endl;
    som.setup();
}

void ofApp::onThinkgearAttention(ofxThinkgearEventArgs& args){
    //attention.value = args.attention;
    eegAttention = args.attention;
    if(eegAttention > eegMeditation){
        //som.setup();
    }
}

void ofApp::onThinkgearMeditation(ofxThinkgearEventArgs& args){
    //meditation.value = args.meditation;
    eegMeditation = args.meditation;
    if(eegMeditation < eegAttention){
        //som.setup();
    }
}

void ofApp::onThinkgearEeg(ofxThinkgearEventArgs& args){
    cout << "delta: " << args.eegDelta << endl;
    delta = args.eegDelta;
    cout << "theta: " << args.eegTheta << endl;
    theta = args.eegTheta;
    cout << "lowalpha: " << args.eegLowAlpha << endl;
    lowalpha = args.eegLowAlpha;
    cout << "highalpha: " << args.eegHighAlpha << endl;
    highalpha = args.eegHighAlpha;
    cout << "lowbeta: " << args.eegLowBeta << endl;
    lowbeta = args.eegLowBeta;
    cout << "highbeta: " << args.eegHighBeta << endl;
    highbeta = args.eegHighBeta;
    cout << "lowgamma: " << args.eegLowGamma << endl;
    lowgamma = args.eegLowGamma;
    cout << "midgamma: " << args.eegMidGamma << endl;
    midgamma = args.eegMidGamma;
    
}

void ofApp::onThinkgearConnecting(ofxThinkgearEventArgs& args){
}

Leave a Reply

Your email address will not be published. Required fields are marked *