Midterm Documentation

The brain is a small mysterious organ. It holds memories, allows for logic and reasoning, controls the body’s internal functions, interprets and elicits responses… For our project we wanted to explore the memory functionality. Our project included two interfaces, the brain and the box. Touching different parts of the brain triggered different memories (audio and video and image files). We tried to make it as believable as possible in terms of physicality so that people will think they are playing with a real brain. The box was used to hold the brain, keep it safe, and emphasize the discovery process. Our piece used continuous input. At all times the brain was sending data that could be seen as a hypnotizing shape-shifting color-blob on the screen. However, when the user explored the brain memories would reveal themselves in the form of visual and audio files. Unexpected juxtaposition of memories occurred when users simultaneously stimulated two or more sensitive areas. The brain does not require user interaction to function, however when it is being handled, memories surface. This discovery process allows the user to see information that would not occur without their interaction and therefore gives them some sense control. The brain is segmented into different functionality and so handling different areas of the brain result in different forms of memories (unique still image, video and audio content). Therefore, while input data is a simple capacity sensor data reading, complex algorithms call to the different media. We chose non-specific generalized videos, images, and audio (watching birds fly, seeing your shadow, driving in a car, stream rising, a crowded street, playing with dogs) as playback material so that each user could relate to it. That way, even though users might not recall personal memories, they are still familiar with the content and have an implicit understanding. When the brain is being handled, the computer displays visual data that is seen by the user. The user reacts to the visual data by making a choice to continue handling the same place, to handle different places or to stop handling altogether. There is a feedback loop in how the visuals are affecting the user’s judgment.

Arduino Code:

#include   CapSense sensor1 = CapSense(3, 2); CapSense sensor2 = CapSense(5, 4); CapSense sensor3 = CapSense(7, 6); CapSense sensor4 = CapSense(9, 8); CapSense sensor5 = CapSense(11, 10); CapSense sensor6 = CapSense(13, 12);  void setup() {  Serial.begin(9600); }  void loop() {  long cap1 = sensor1.capSense(15);  long cap2 = sensor2.capSense(15);  long cap3 = sensor3.capSense(15);  long cap4 = sensor4.capSense(15);  long cap5 = sensor5.capSense(15);  long cap6 = sensor6.capSense(15);  Serial.print(cap1);  Serial.print("\t");  Serial.print(cap2);  Serial.print("\t");  Serial.print(cap3);  Serial.print("\t");  Serial.print(cap4);  Serial.print("\t");  Serial.print(cap5);  Serial.print("\t");  Serial.print(cap6);  Serial.print("\n"); }

Processing Code:

/*Media Files are taken from Tamara Pokrupa's home videos, Rola Kuidir's images,

generic audio files and apple screensaver video*/

//Arduino

import processing.serial.*;

Serial serial;

int data1;

int data2;

int data3;

int data4;

int data5;

int data6;

//Video

import processing.video.*;

Movie movie, movie1, movie2, movie3, movie4;

//Audio

import ddf.minim.*;

import ddf.minim.signals.*;

import ddf.minim.analysis.*;

import ddf.minim.effects.*;

Minim minim;

AudioPlayer one;

AudioPlayer two;

AudioPlayer three;

AudioPlayer four;

AudioPlayer five;

AudioPlayer six;

//Images

PImage img1;

PImage img2;

//Cases

int signal= 0;

void setup () {

size(500,300,P2D);

//setup Arduino

println(Serial.list()[0]);

serial = new Serial(this, Serial.list()[0], 9600);

serial.clear();

serial.bufferUntil('\n');

//setup Video

movie= new Movie(this, "default2.mov");

movie1=new Movie(this, "dog.m4v");

movie2=new Movie(this, "birds.m4v");

movie3= new Movie(this, "cold.m4v");

movie4= new Movie(this, "corwded.m4v");

movie.loop();

movie1.loop();

movie2.loop();

movie3.loop();

movie4.loop();

//setup Audio

minim = new Minim(this);

one=minim.loadFile("Road.mp3");

two=minim.loadFile("dog.mp3");

three=minim.loadFile("birds.mp3");

four=minim.loadFile("cold.mp3");

five=minim.loadFile("crowded.mp3");

six=minim.loadFile("Footsteps.aif");

one.loop();

two.loop();

three.loop();

four.loop();

five.loop();

six.loop();

//setup Image

img1=loadImage("Road2.jpg");

img2=loadImage("walkin.jpg");

}

//Arduino readings

void serialEvent(Serial serial) {

String packet = serial.readStringUntil('\n');

// Now we must do a bit more work to separate the two elements

// of our packet, which has the form

// \t\n

packet = packet.trim(); // removes the '\n' at the end

String[] parts = packet.split("\t"); // split the string in two where the tab character is

try {

data1 = Integer.parseInt(parts[0]);

data2 = Integer.parseInt(parts[1]);

data3 = Integer.parseInt(parts[2]);

data4 = Integer.parseInt(parts[3]);

data5 = Integer.parseInt(parts[4]);

data6 = Integer.parseInt(parts[5]);

}

catch(Exception e) {

println("There was a problem with the packet: " + packet);

}

}

void draw () {

background(0);

// data from Arduino to call specific function

if(signal==0){

default1();

}else if (signal==1){

signal1();

}else if(signal==2){

signal2();

}else if(signal==3){

signal3();

}else if(signal==4){

signal4();

}else if(signal==5){

signal5();

}else if(signal==6){

signal6();

}

//Read signals from arduino

if(data1>10){

signal=1;

}else if(data2>10){

signal=2;

}else if (data3>10){

signal=3;

}else if(data4>10){

signal=4;

}else if(data5>10){

signal=5;

}else if(data6>10){

signal=6;

}else{

signal=0;

}

}

void default1(){

//Image, Video

one.mute();

two.mute();

three.mute();

four.mute();

five.mute();

six.mute();

//Image, Video

movie1.pause();

movie2.pause();

movie3.pause();

movie4.pause();

movie.play();

image(movie,0,0,width,height);

}

void signal1(){

//Audio

one.unmute();

two.mute();

three.mute();

four.mute();

five.mute();

six.mute();

//Image, Video

image(img1,0,0,width,height);

movie.pause();

movie1.pause();

movie2.pause();

movie3.pause();

movie4.pause();

}

void signal2(){

//Audio

one.mute();

two.unmute();

three.mute();

four.mute();

five.mute();

six.mute();

//Image, Video

movie.pause();

movie2.pause();

movie3.pause();

movie4.pause();

movie1.play();

image(movie1,0,0,width,height);

}

void signal3(){

//Audio

one.mute();

two.mute();

three.unmute();

four.mute();

five.mute();

six.mute();

////Image, Video

movie.pause();

movie1.pause();

movie3.pause();

movie4.pause();

image(movie2,0,0,width,height);

movie2.play();

}

void signal4(){

//Audio

one.mute();

two.mute();

three.mute();

four.unmute();

five.mute();

six.mute();

//Image, Video

movie.pause();

movie1.pause();

movie2.pause();

movie4.pause();

image(movie3,0,0,width,height);

movie3.play();

}

void signal5(){

//Audio

one.mute();

two.mute();

three.mute();

four.mute();

five.unmute();

six.mute();

//Image, Video

movie.pause();

movie1.pause();

movie2.pause();

movie3.pause();

image(movie4,0,0,width,height);

movie4.play();

}

void signal6(){

//Audio

one.mute();

two.mute();

three.mute();

four.mute();

five.mute();

six.unmute();

//Image, Video

movie.pause();

movie1.pause();

movie2.pause();

movie3.pause();

movie4.pause();

image(img2,0,0,width,height);

}

void movieEvent(Movie movie){

movie.read();

movie1.read();

movie2.read();

movie3.read();

movie4.read();

}

void stop(){

// always stop Minim before exiting.

minim.stop();

one.close(); // always close Minim audio classes when you are done with them

two.close();

three.close();

four.close();

five.close();

six.close(); // always close Minim audio classes when you are done with them

super.stop();

}