Context: One of the main contributions of technology is mediation: technology mediates the agency of both disabled and fully enabled humans in the world, rebuilding their relation with the environment.
Problem: For visually impaired people, many applications have been developed that scan and recognise objects in space for visually impaired people. This way, the user can perceive spatial information without employing touch. The visual information is translated into auditory words, which describe an object or a spatial condition.
This method facilitates the daily life for people with visual impairment. However, it is inadequate when it comes to the perception of art. A colourful painting cannot be described through a sequence of colour matches that may bear no meaning to people with congenital vision disability.
Solution: ‘Hearing the colours is a wearable interface that turns the experience of colours and visual art into an audio performance. The wearable prompts the user to move their hand and ‘grasp’ the sounds of the colours in a similar way with that they execute to ‘grasp’ the Braille dots for reading.
The interface plays the role of a sensory substitution device (SSD). After practice, the user learns to stimulate spontaneously the experiential quality of “seeing a color” through a new set of sensoricognitive skills. . This changes the classical definition of sensory modalities and contributes to the emergence of a form of “artificial synaesthesia”.
The interface is designed to transform the static observation of art into an interplay of visuals, sound and body movement. However, hearing the colors through a wearable device should not only be constrained to the spectrum of artwork. It contains the potential to turn the body into a scanning device, that moves and interacts with the space itself, even with other human beings.
Color as a R:G:B and H:S:B code
Computational Logic:
Demonstration Demo:
Circuit design:
Code:
#include <Wire.h>
#include "Adafruit_TCS34725.h"
/* Initialise with specific int time (700ms) and gain(1x) values
Integration time = how often it reads the colors
Longer integration times can be used for increased sensitivity at low light levels.
Valid integration times are: 2.4MS, 24MS, 50MS, 101MS, 154MS, 700MS
Sets the gain of the ADC to control the sensitivity of the sensor.
Valid gain settings are: 1x(no gain), 4x, 16x, 60x
*/
Adafruit_TCS34725 myRGB = Adafruit_TCS34725(TCS34725_INTEGRATIONTIME_50MS, TCS34725_GAIN_4X);
int R = 0;
int G = 0;
int B = 0;
int hsv [3];
const int speaker = 11;
int pitch; int duration; int delayT;
/*********************************************************************************************/
void setup() {
Serial.begin(9600);
if (myRGB.begin()) {
Serial.println("The colorSensor is found");
} else {
Serial.println("No colorSensori is found");
while (1);
}
Serial.println("*****************************************************");
Serial.println();
}
/*********************************************************************************************/
void loop() {
getRGBValues();
rgb2Hsv(R, G, B, hsv);
colors2Sounds();
tone(speaker, pitch, duration);
delay(delayT);
} // loop
/*********************************************************************************************/
void colors2Sounds() {
if (hsv[0] <= 1) {
pitch = 100;
duration = 400;
delayT = 0;
Serial.println("red");
} else if (hsv[0] <= 5) {
pitch = 300;
duration = 400;
delayT = 50;
Serial.println("purple");
} else if (hsv[0] <= 13) {
pitch = 500;
duration = 400;
delayT = 100;
Serial.println("orange");
} else if (hsv[0] <= 17) {
pitch = 1000;
duration = 400;
delayT = 180;
Serial.println("yellow");
} else if (hsv[0] <= 25) {
pitch = 1800;
duration = 400;
delayT = 230;
Serial.println("green");
} else if (hsv[0] <= 33) {
pitch = 2400;
duration = 400;
delayT = 300;
Serial.println("greenish blue");
} else if (hsv[0] <= 55) {
pitch = 3000;
duration = 400;
delayT = 400;
Serial.println("blue");
}
}
void getRGBValues() {
float _red, _green, _blue;
myRGB.setInterrupt(false);
delay(60);
myRGB.getRGB(&_red, &_green, &_blue);
myRGB.setInterrupt(true);
//Print RGB values
/* Serial.print("Red: "); Serial.print(_red); Serial.print(" *** ");
Serial.print("Green: "); Serial.print(_green); Serial.print(" *** ");
Serial.print("Blue: "); Serial.print(_blue); Serial.print(" *** ");
Serial.println(); */
R = _red;
G = _green;
B = _blue;
} // getRGBValues
void rgb2Hsv(byte r, byte g, byte b, int hsv[]) {
float rd = (float) r / 255;
float gd = (float) g / 255;
float bd = (float) b / 255;
float max = threeway_max(rd, gd, bd), min = threeway_min(rd, gd, bd);
float h, s, v = max;
float d = max - min;
s = max == 0 ? 0 : d / max;
if (max == min) {
h = 0; // achromatic
} else {
if (max == rd) {
h = (gd - bd) / d + (gd < bd ? 6 : 0);
} else if (max == gd) {
h = (bd - rd) / d + 2;
} else if (max == bd) {
h = (rd - gd) / d + 4;
}
h /= 6;
}
hsv[0] = int(h * 100);
hsv[1] = int(s * 100);
hsv[2] = int(v * 100);
Serial.println((String)"Hue: " + hsv[0] + " Saturation: " + hsv[1] + " Value: " + hsv[2]);
}
float threeway_max(float r, float g, float b) {
float res;
res = r;
if (g > res) {
res = g;
}
if (b > res) {
res = b;
}
return res;
}
float threeway_min(float r, float g, float b) {
float res;
res = r;
if (g < res) {
res = g;
}
if (b < res) {
res = b;
}
return res;
}