Generating mouth shapes using simple lookup table

.idea/LipSync.iml

@@ -139,12 +139,16 @@
       <sourceFolder url="file://$MODULE_DIR$/src/centiseconds.cpp" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/centiseconds.h" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/main.cpp" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/mouth_animation.cpp" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/mouth_animation.h" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/Phone.cpp" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/Phone.h" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/phone_extraction.cpp" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/phone_extraction.h" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/platform_tools.h" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/platform_tools_win.cpp" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/Shape.cpp" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/Shape.h" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/tools.cpp" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/tools.h" isTestSource="false" />
     </content>

CMakeLists.txt

@@ -14,7 +14,7 @@ set(Boost_USE_STATIC_RUNTIME ON) # Use static C++ runtime
 find_package(Boost REQUIRED COMPONENTS filesystem locale system)
 include_directories(${Boost_INCLUDE_DIRS})
 
-set(SOURCE_FILES src/main.cpp src/audio_input/WaveFileReader.cpp src/audio_input/WaveFileReader.h src/audio_input/ChannelDownmixer.cpp src/audio_input/ChannelDownmixer.h src/audio_input/AudioStream.h src/audio_input/SampleRateConverter.cpp src/audio_input/SampleRateConverter.h src/audio_input/wave_file_writing.cpp src/audio_input/wave_file_writing.h src/audio_input/io_tools.h src/platform_tools.h src/phone_extraction.cpp src/phone_extraction.h src/Phone.cpp src/Phone.h src/centiseconds.cpp src/centiseconds.h src/tools.cpp src/tools.h)
+set(SOURCE_FILES src/main.cpp src/audio_input/WaveFileReader.cpp src/audio_input/WaveFileReader.h src/audio_input/ChannelDownmixer.cpp src/audio_input/ChannelDownmixer.h src/audio_input/AudioStream.h src/audio_input/SampleRateConverter.cpp src/audio_input/SampleRateConverter.h src/audio_input/wave_file_writing.cpp src/audio_input/wave_file_writing.h src/audio_input/io_tools.h src/platform_tools.h src/phone_extraction.cpp src/phone_extraction.h src/Phone.cpp src/Phone.h src/centiseconds.cpp src/centiseconds.h src/tools.cpp src/tools.h src/Shape.cpp src/Shape.h src/mouth_animation.cpp src/mouth_animation.h)
 if(WIN32)
 	set(SOURCE_FILES "${SOURCE_FILES};src/platform_tools_win.cpp")
 else()

src/Phone.cpp

@@ -34,3 +34,6 @@ string phoneToString(Phone phone) {
 	return (it != phonesByName.right.end()) ? it->second : phoneToString(Phone::Unknown);
 }
 
+std::ostream &operator<<(std::ostream &stream, const Phone phone) {
+	return stream << phoneToString(phone);
+}

src/Phone.h

@@ -74,5 +74,6 @@ Phone stringToPhone(const std::string& s);
 
 std::string phoneToString(Phone phone);
 
+std::ostream& operator <<(std::ostream& stream, const Phone phone);
 
 #endif //LIPSYNC_PHONE_H

src/Shape.cpp

@@ -0,0 +1,10 @@
+#include "Shape.h"
+
+std::string shapeToString(Shape shape) {
+	char c = 'A' + static_cast<int>(shape);
+	return std::string(&c, 1);
+}
+
+std::ostream &operator<<(std::ostream &stream, const Shape shape) {
+	return stream << shapeToString(shape);
+}

src/Shape.h

@@ -0,0 +1,24 @@
+#ifndef LIPSYNC_SHAPE_H
+#define LIPSYNC_SHAPE_H
+
+#include <string>
+
+// The classic Hanna-Barbera mouth shapes A-F phus the common supplements G-H
+// For reference, see http://sunewatts.dk/lipsync/lipsync/article_02.php
+// For visual examples, see https://flic.kr/s/aHsj86KR4J. Their shapes "BMP".."L" map to A..H.
+enum class Shape {
+	A,	// Closed mouth (silence, M, B, P)
+	B,	// Clenched teeth (most vowels, m[e]n)
+	C,	// Mouth slightly open (b[ir]d, s[ay], w[i]n...)
+	D,	// Mouth wide open (b[u]t, m[y], sh[ou]ld...)
+	E,	// h[ow]
+	F,	// Pout ([o]ff, sh[ow])
+	G,	// F, V
+	H	// L
+};
+
+std::string shapeToString(Shape shape);
+
+std::ostream& operator <<(std::ostream& stream, const Shape shape);
+
+#endif //LIPSYNC_SHAPE_H

src/main.cpp

@@ -1,6 +1,7 @@
 #include <iostream>
 #include "audio_input/WaveFileReader.h"
 #include "phone_extraction.h"
+#include "mouth_animation.h"
 #include "platform_tools.h"
 
 using std::exception;
@@ -42,8 +43,14 @@ int main(int argc, char *argv[]) {
 		// Detect phones
 		std::map<centiseconds, Phone> phones = detectPhones(std::move(audioStream));
 
+		// Generate mouth shapes
+		std::map<centiseconds, Shape> shapes = animate(phones);
+
 		for (auto &pair : phones) {
-			std::cout << pair.first << ": " << phoneToString(pair.second) << "\n";
+			std::cout << pair.first << ": " << pair.second << "\n";
+		}
+		for (auto &pair : shapes) {
+			std::cout << pair.first << ": " << pair.second << "\n";
 		}
 
 		return 0;

src/mouth_animation.cpp

@@ -0,0 +1,75 @@
+#include "mouth_animation.h"
+
+using std::map;
+
+Shape getShape(Phone phone) {
+	switch (phone) {
+		case Phone::None:
+		case Phone::P:
+		case Phone::B:
+		case Phone::M:
+			return Shape::A;
+
+		case Phone::Unknown:
+		case Phone::EH:
+		case Phone::T:
+		case Phone::D:
+		case Phone::K:
+		case Phone::G:
+		case Phone::CH:
+		case Phone::JH:
+		case Phone::TH:
+		case Phone::DH:
+		case Phone::S:
+		case Phone::Z:
+		case Phone::SH:
+		case Phone::ZH:
+		case Phone::N:
+		case Phone::NG:
+		case Phone::R:
+			return Shape::B;
+
+		case Phone::IY:
+		case Phone::IH:
+		case Phone::EY:
+		case Phone::ER:
+		case Phone::HH:
+		case Phone::Y:
+			return Shape::C;
+
+		case Phone::UW:
+		case Phone::UH:
+		case Phone::AH:
+		case Phone::AE:
+		case Phone::AY:
+		case Phone::OY:
+		case Phone::W:
+			return Shape::D;
+
+		case Phone::AW:
+			return Shape::E;
+
+		case Phone::AO:
+		case Phone::AA:
+		case Phone::OW:
+			return Shape::F;
+
+		case Phone::F:
+		case Phone::V:
+			return Shape::G;
+
+		case Phone::L:
+			return Shape::H;
+
+		default:
+			throw std::runtime_error("Unexpected Phone value.");
+	}
+}
+
+map<centiseconds, Shape> animate(const map<centiseconds, Phone> &phones) {
+	map<centiseconds, Shape> shapes;
+	for (auto& pair : phones) {
+		shapes[pair.first] = getShape(pair.second);
+	}
+	return shapes;
+}

src/mouth_animation.h

@@ -0,0 +1,11 @@
+#ifndef LIPSYNC_MOUTH_ANIMATION_H
+#define LIPSYNC_MOUTH_ANIMATION_H
+
+#include <map>
+#include "Phone.h"
+#include "centiseconds.h"
+#include "Shape.h"
+
+std::map<centiseconds, Shape> animate(const std::map<centiseconds, Phone>& phones);
+
+#endif //LIPSYNC_MOUTH_ANIMATION_H