Author: mseo1@andrew.cmu.edu

sketch

//Chrisitne Seo
//mseo1@andrew.cmu.edu
//Section C
//Final Project

//items, where they are placed
var smileX = 200, smile, smileY = 0;
var clover, cloverX = 100, cloverY = 0;
var drop, dropX = 250, dropY = 0;
var cat, catX = 495, catY = 0;
var fire, fireX = 380, fireY = 0;
var flower, flowerX = 430, flowerY = 0;
var heart, heartX = 330, heartY = 0;
var mad, madX = 20, madY = 0;
var star, starX = 140, starY = 0;
var thunder, thunderX = 150, thunderY = 0;
var startY = 0;

var beat1, beat2, beat3, beat4, beat5, beat7, beat8, beat9, beatS;

//speed that the items want to go
var directionY = 1, directionX = 0, direction1Y = 0.7, direction2Y = 2,direction3Y = 2.5, direction3Y = 2.5, direction4Y = 2.1, direction5Y = 0.4, direction6Y = 0.2, direction7Y = 1.5, direction8Y = 1.2, direction9Y = 0.2;

//no repetition of sounds when touched once
var lastFrameTouch, lastFrameTouch2, lastFrameTouch3, lastFrameTouch4, lastFrameTouch5, lastFrameTouch6, lastFrameTouch7, lastFrameTouch8, lastFrameTouch9;

//sound trackers
var ctracker, mic;
var volume = 0;
var currScene = 0;

//recording
var recorder;
var soundFile;
var state = 0;

//color of eyebrows
var r = 176;
var g = 196;
var b = 222;

function preload(){ //beats that each item plays
  beat1 = loadSound("beats/1.mp3");
  beat2 = loadSound("beats/2.mp3");
  beat3 = loadSound("beats/3.mp3");
  beat4 = loadSound("beats/4.mp3");
  beat5 = loadSound("beats/5.mp3");
  beat7 = loadSound("beats/7.mp3");
  beat8 = loadSound("beats/8.mp3");
  beat9 = loadSound("beats/9.mp3");
  beatS  = loadSound("beats/swoosh.mp3");
}

function setup() {
  
  // setup camera capture
  var videoInput = createCapture(VIDEO);
  videoInput.size(550, 500);
  videoInput.position(0, 0);

  // setup canvas
  var cnv = createCanvas(550, 500);
  cnv.position(0, 0);

  // setup tracker
  ctracker = new clm.tracker();
  ctracker.init(pModel);
  ctracker.start(videoInput.elt);

  // Create an Audio input
  mic = new p5.AudioIn();
  mic.start();
  
  //recording
  recorder = new p5.SoundRecorder();
  recorder.setInput(mic);
  soundFile = new p5.SoundFile();
  
  //loading the items
  smile = loadImage("images/smile.png");
  clover = loadImage("images/clover.png");
  drop = loadImage("images/drop.png");
  cat = loadImage("images/cat.png");
  fire = loadImage("images/fire.png");
  heart = loadImage("images/heart.png");
  flower = loadImage("images/flower.png");
  mad = loadImage("images/mad.png");
  star = loadImage("images/star.png");
  
  //making sure each item is false when it didn't touch the face
  lastFrameTouch = false;
  lastFrameTouch2 = false;
  lastFrameTouch3 = false;
  lastFrameTouch4 = false;
  lastFrameTouch5 = false;
  lastFrameTouch6 = false;
  lastFrameTouch7 = false;
  lastFrameTouch8 = false;
  lastFrameTouch9 = false;
  
}

function draw() {
  
  //playing through by "scenes"
  if(currScene == 0){
  	startScene();
  } else if (currScene == 1){
    camScene();  
		}
}

function startScene(){ 
  
  //frame
  push();
  noFill();
  stroke("PaleVioletRed");
  strokeWeight(50);
  rect(0,0,width,height);
  pop();
  
  //start button
  stroke("LightSteelBlue");
  strokeWeight(5);
  fill(0);
  rect(200,365,170,100,60);
  push();
  fill("PaleVioletRed");
  strokeWeight(3);
  textSize(50);
  textStyle(BOLD);
  stroke("PapayaWhip");
  text("start",232,430);

  //intstructions
  fill(0);
  textSize(20);
  text("Catch each item",310,70);
  text("with your eyebrows",310,100);
  text("in order to make",310,130);
  text("your own soundtrack!",310,160);
  textSize(13);
  stroke("LightSteelBlue");
  text("*fyi: need to refresh if",350,190);
  text("face detection jitters",350,210);
  stroke("PaleVioletRed");
  text("*play with speakers",350,230);
  pop();
  
}

function camScene(){
  
  //face tracker codes
  clear();
 
  //getting mic
  var v = mic.getLevel();
  
  //getting volume
  volume += (v-volume)/3;

  var detectionScore = ctracker.getScore();
  	
 	if (detectionScore > 0) { 
    //location points of the faces  
  	var positions = ctracker.getCurrentPosition();
  	var leftBrowX = positions [20][0];
  	var leftBrowY = positions [20][1];  
    var rightBrowX = positions [16][0];
    var rightBrowY = positions [16][1];
    var faceLeftX = positions [1][0];
    var faceLeftY = positions [1][1];
    var faceRightX = positions [13][0];
    var faceRightY = positions [13][1];
    var noseX = positions [62][0]
    var noseY = positions [62][1];
    var eyeLeftX = positions [27][0];
    var eyeLeftY = positions [27][1];
    var eyeRightX = positions [32][0];
    var eyeRightY = positions [32][1];
    //corresponding the face feature sizes by volume
    var mouthSize = map(volume, 0,0.3, 70, 100);
    var eyeSize = map(volume,0,0.3,45,75);
    var eyeballSize = map(volume,0,0.3,40,70);
    var noseSize = map(volume,0,0.3,25,55);
    var eyebrowSize = map(volume,0,0.3,20,30);
    
    rect(5,5,230,40);
    fill(0);  
    textSize(15);  
    text('Double click to pause & record', 18, 25);
    
  //smile sounds
  if (smileX > leftBrowX - 50 &  smileX < leftBrowX + 50 && smileY > leftBrowY - 20 && smileY < leftBrowY + 20){
    directionY = 0; //stop when it touches the eyebrow until brow moves again
    r = 255; //change colors of eyebrows in relation to item
    g = 255;
    b = 0;
    if(! lastFrameTouch2){ 
      beat2.loop();  
      lastFrameTouch2 = true; //play sound once
      r = 255; //change colors of eyebrows in relation to item
      g = 255;
      b = 0;  
    } //same as about but for right brow placements
  } else if (smileX > rightBrowX - 50 &  smileX < rightBrowX + 50 && smileY > rightBrowY - 20 && smileY < rightBrowY + 20){
    	directionY = 0;  
      if(! lastFrameTouch2){ 
      	beat2.loop();  
        lastFrameTouch2 = true;
      } //if item stops touching the face, continue falling down
    } else { 
        directionY = 1.5;
      }
    
	//fire sounds
  if (fireX > leftBrowX - 50 &  fireX < leftBrowX + 50 && fireY > leftBrowY - 20 && fireY < leftBrowY + 20){
    direction4Y = 0; //stop when it touches the eyebrow until brow moves again
    r = 255; //change colors of eyebrows in relation to item
    g = 125;
    b = 65;  
    if(! lastFrameTouch){
    	beat1.loop();  
      lastFrameTouch = true; //play sound once
    } //same as about but for right brow placements
  } else if (fireX > rightBrowX - 50 &  fireX < rightBrowX + 50 && fireY > rightBrowY - 20 && fireY < rightBrowY + 20){
    direction4Y = 0;  
    r = 255;
    g = 125;
    b = 65; 
    if(! lastFrameTouch){
    	beat1.loop();  
      lastFrameTouch = true;
    } //if item stops touching the face, continue falling down
  } else {
      direction4Y = 1.6;
    }
    
    //clover sounds
	if (cloverX > leftBrowX - 50 &  cloverX < leftBrowX + 50 && cloverY > leftBrowY - 20 && cloverY < leftBrowY + 20){
  	direction1Y = 0; //stop when it touches the eyebrow until brow moves again
    r = 75; //change colors of eyebrows in relation to item
    g = 185;
    b = 105; 
    if(! lastFrameTouch3){
    	beat3.loop();  
      lastFrameTouch3 = true; //play sound once
    } //same as about but for right brow placements
	} else if (cloverX > rightBrowX - 50 &  cloverX < rightBrowX + 50 && cloverY > rightBrowY - 20 && cloverY < rightBrowY + 20){
    direction1Y = 0; 
    r = 75;
    g = 185;
    b = 105; 
    if(! lastFrameTouch3){
    	beat3.loop();  
      lastFrameTouch3 = true;
    } //if item stops touching the face, continue falling down
  } else {
  	direction1Y = 0.9;
    }
    
    //drop sounds
	if (dropX > leftBrowX - 50 &  dropX < leftBrowX + 50 && dropY > leftBrowY - 20 && dropY < leftBrowY + 20){
  	direction2Y = 0; //stop when it touches the eyebrow until brow moves again
    r = 130; //change colors of eyebrows in relation to item
    g = 200;
    b = 255; 
    if(! lastFrameTouch4){
    	beat4.loop();  
      lastFrameTouch4 = true; //play sound once
    } //same as about but for right brow placements
  } else if (dropX > rightBrowX - 50 &  dropX < rightBrowX + 50 && dropY > rightBrowY - 20 && dropY < rightBrowY + 20){
    direction2Y = 0;  
    r = 130;
    g = 200;
    b = 255; 
		if(! lastFrameTouch4){
  		beat4.loop();  
      lastFrameTouch4 = true;
    } //if item stops touching the face, continue falling down
	} else {
    direction2Y = 0.73;
    }
    
   //cat sounds
	if (catX > leftBrowX - 50 &  catX < leftBrowX + 50 && catY > leftBrowY - 20 && catY < leftBrowY + 20){
		direction3Y = 0; //stop when it touches the eyebrow until brow moves again
    r = 160; //change colors of eyebrows in relation to item
    g = 160;
    b = 160;      
    if(! lastFrameTouch5){
    	beat5.loop();  
      lastFrameTouch5 = true; //play sound once
    } //same as about but for right brow placements
	} else if (catX > rightBrowX - 50 &  catX < rightBrowX + 50 && catY > rightBrowY - 20 && catY < rightBrowY + 20){
    direction3Y = 0;  
    r = 160;
    g = 160;
    b = 160;   
		if(! lastFrameTouch5){
    	beat5.loop();  
      lastFrameTouch5 = true;
    } //if item stops touching the face, continue falling down
	} else {
    direction3Y = 0.65;
    }

//flower sounds
	if (flowerX > leftBrowX - 50 & flowerX < leftBrowX + 50 && flowerY > leftBrowY - 20 && flowerY < leftBrowY + 20){
  	direction5Y = 0; //stop when it touches the eyebrow until brow moves again
    r = 250; //change colors of eyebrows in relation to item
    g = 250;
    b = 250;
    if(! lastFrameTouch6){
    	beat8.loop();  
      lastFrameTouch6 = true; //play sound once
    } //same as about but for right brow placements
	} else if (flowerX > rightBrowX - 50 & flowerX < rightBrowX + 50 && flowerY > rightBrowY - 20 && flowerY < rightBrowY + 20){
    direction5Y = 0;  
    r = 250;
    g = 250;
    b = 250;
		if(! lastFrameTouch6){
      beat8.loop();  
      lastFrameTouch6 = true;
		} //if item stops touching the face, continue falling down
	} else {
    direction5Y = 0.8;
    }
    
   //heart sounds
	if (heartX > leftBrowX - 50 &  heartX < leftBrowX + 50 && heartY > leftBrowY - 20 && heartY < leftBrowY + 20){
		direction6Y = 0; //stop when it touches the eyebrow until brow moves again
    r = 200; //change colors of eyebrows in relation to item
    g = 0;
    b = 50;
    if(! lastFrameTouch7){
    	beat7.loop();  
    	lastFrameTouch7 = true; //play sound once
    } //same as about but for right brow placements
	} else if (heartX > rightBrowX - 50 & heartX < rightBrowX + 50 && heartY > rightBrowY - 20 && heartY < rightBrowY + 20){
    direction6Y = 0; 
    r = 200;
    g = 0;
    b = 50;
		if(! lastFrameTouch7){
      beat7.loop();  
      lastFrameTouch7 = true;
    } //if item stops touching the face, continue falling down
	} else {
    direction6Y = 1.65;
    }
    
    //mad sounds
	if (madX > leftBrowX - 50 & madX < leftBrowX + 50 && madY > leftBrowY - 20 && madY < leftBrowY + 20){
  	direction7Y = 0; //stop when it touches the eyebrow until brow moves again
    r = 250; //change colors of eyebrows in relation to item
    g = 0;
    b = 0;
    if(! lastFrameTouch8){
      beat9.loop();  
      lastFrameTouch8 = true; //play sound once
    } //same as about but for right brow placements
	} else if (madX > rightBrowX - 50 & madX < rightBrowX + 50 && madY > rightBrowY - 20 && madY < rightBrowY + 20){
    direction7Y = 0;  
    r = 250;
    g = 0;
    b = 0;
		if(! lastFrameTouch8){
      beat9.loop();  
      lastFrameTouch8 = true;
    } //if item stops touching the face, continue falling down
  } else {
    direction7Y = 1.3;
    }
    
    //star sounds
	if (starX > leftBrowX - 50 & starX < leftBrowX + 50 && starY > leftBrowY - 20 && starY < leftBrowY + 20){
    direction8Y = 0; //stop when it touches the eyebrow until brow moves again
    r = 240; //change colors of eyebrows in relation to item
    g = 220;
    b = 50;
    if(! lastFrameTouch9){
      beatS.loop();  
      lastFrameTouch9 = true; //play sound once
    } //same as about but for right brow placements
	} else if (starX > rightBrowX - 50 & starX < rightBrowX + 50 && starY > rightBrowY - 20 && starY < rightBrowY + 20){
    direction8Y = 0;  
    r = 240;
    g = 220;
    b = 50;  
		if(! lastFrameTouch9){
      beatS.loop();  
      lastFrameTouch9 = true;
    } //if item stops touching the face, continue falling down
  } else {
      direction8Y = 0.45;
    }
    
  //eyebrow
	stroke(r,g,b);
  fill("PapayaWhip");
  strokeWeight(7);
  rectMode(RADIUS);
  rect(leftBrowX,leftBrowY,eyebrowSize + 20,eyebrowSize-20);
  rect(rightBrowX,rightBrowY,eyebrowSize + 20,eyebrowSize-20);
    
  //eyes
  stroke("PapayaWhip");
  strokeWeight(2);
  fill(0);
  ellipse(eyeLeftX,eyeLeftY,eyeSize,eyeSize);
  ellipse(eyeRightX,eyeRightY,eyeSize,eyeSize);
  ellipse(positions[27][0],positions[27][1],eyeballSize,eyeballSize); 
  ellipse(positions[32][0],positions[32][1],eyeballSize,eyeballSize);

  //nose
  stroke("PaleVioletRed");
  fill("LightSteelBlue");
  ellipse(positions[62][0],positions[62][1],noseSize,noseSize+20);
        	
  //mouth
  stroke("PapayaWhip");
  fill("PaleVioletRed");
  arc(positions[47][0],positions[47][1],mouthSize,mouthSize,0,PI);
  }
  
  //placing item and moving it downwards and back up when it reaches bottom
  image(smile,smileX,smileY,50,50);
  smileX += directionX;
  smileY += directionY;
  	if (smileY > 460){
   		smileY = 0;
  	}
  
  //placing item and moving it downwards and back up when it reaches bottom
  image(clover,cloverX,cloverY,50,50);
  cloverX += directionX;
  cloverY += direction1Y;
    if (cloverY > 460){
    cloverY = 0;
  }
  
  //placing item and moving it downwards and back up when it reaches bottom
  image(drop,dropX,dropY,50,50);
  dropX += directionX;
  dropY += direction2Y;
    if (dropY > 460){
    dropY = 0;
  }
  
  //placing item and moving it downwards and back up when it reaches bottom
  image(cat,catX,catY,40,40);
  catX += directionX;
  catY += direction3Y;
    if (catY > 460){
    catY = 0;
  }
  
  //placing item and moving it downwards and back up when it reaches bottom
  image(fire,fireX,fireY,50,55);
  fireX += directionX;
  fireY += direction4Y;
    if (fireY > 460){
    fireY = 0;
  }
  
  //placing item and moving it downwards and back up when it reaches bottom
  image(flower,flowerX,flowerY,35,35);
  flowerX += directionX;
  flowerY += direction5Y;
    if (flowerY > 460){
    flowerY = 0;
  }
  
  //placing item and moving it downwards and back up when it reaches bottom
  image(heart,heartX,heartY,40,40);
  heartX += directionX;
  heartY += direction6Y;
    if (heartY > 460){
    heartY = 0;
  }

  //placing item and moving it downwards and back up when it reaches bottom
  image(star,starX,starY,40,40);
  starX += directionX;
  starY += direction8Y;
    if (starY > 460){
    starY = 0;
  }

  //placing item and moving it downwards and back up when it reaches bottom
  image(mad,madX,madY,40,40);
  madX += directionX;
  madY += direction7Y;
    if (madY > 460){
    madY = 0;
  }
}

function mousePressed(){
  //when mouse pressed, it switches scenes 
  
  if (currScene == 0){
    if (mouseX > 100 & mouseX < 500 && mouseY > 100 && mouseY < 300);
    	currScene =1;
    
  	} else if (currScene == 1){
    if (mouseX > 0 & mouseX < width && mouseY > 0 && mouseY < height);
    	currScene = 2;
      
	} else if (currScene == 2){
    if (state === 0) {
      // Tell recorder to record to a p5.SoundFile which we will use for playback
      recorder.record(soundFile);
      
      //buttons
      fill("LightSteelBlue");    	
      rect(5,5,230,40);
      fill(0);
      textSize(15);    
      text('Recording now! Click to stop.', 18, 25);
      state++;
  } else if (state === 1) {
      recorder.stop(); // stop recorder, and send the result to soundFile
      rect(5,5,288,40);
      fill("PapayaWhip");
      noStroke();
      text('Recording stopped. Click to play & save', 18, 25);
      state++;
  } else if (state === 2) {
      soundFile.play(); // play the result!
      saveSound(soundFile, 'mySound.wav'); // save file
      state++;
    }
	}
}

Caption: Video documentation of the final project

I realized that the WordPress does not run my project, so I provided a live online link to run my project: https://editor.p5js.org/mseo1/full/rkD_gLl0Q

For my final project, I was successfully able to perform what I wanted to display by adding audio (beats) and creating an instrument. This project is a fun game like instrument that lets you play beats (which I made my own beats using Garageband) with the items that are falling down. The beats are played when the items are touched with your eyebrows, which change colors corresponding with the color of the items. Additionally, the face detector adds an interesting visual addition to the project. The elements on the face (eyes, brows, nose, and mouth) increase/decrease in correlation to the sound of the audio, so you must run it with speakers, not headphones. The face recognition element was found on GitHub’s Javascript Library.  Finally, you can record the audio by clicking the button on the top of the screen, and download the file. Overall, this was a fun project to perform, as well as an interesting element to my portfolio.

Zip File: Seo Final