Generating mouth shapes using simple lookup table

This commit is contained in:
Daniel Wolf 2015-11-20 22:20:19 +01:00
parent 994e2be314
commit 2ef99119b0
9 changed files with 137 additions and 2 deletions

View File

@ -139,12 +139,16 @@
<sourceFolder url="file://$MODULE_DIR$/src/centiseconds.cpp" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/src/centiseconds.cpp" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/centiseconds.h" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/src/centiseconds.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main.cpp" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/src/main.cpp" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/mouth_animation.cpp" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/mouth_animation.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/Phone.cpp" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/src/Phone.cpp" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/Phone.h" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/src/Phone.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/phone_extraction.cpp" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/src/phone_extraction.cpp" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/phone_extraction.h" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/src/phone_extraction.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/platform_tools.h" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/src/platform_tools.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/platform_tools_win.cpp" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/src/platform_tools_win.cpp" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/Shape.cpp" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/Shape.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/tools.cpp" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/src/tools.cpp" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/tools.h" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/src/tools.h" isTestSource="false" />
</content> </content>

View File

@ -14,7 +14,7 @@ set(Boost_USE_STATIC_RUNTIME ON) # Use static C++ runtime
find_package(Boost REQUIRED COMPONENTS filesystem locale system) find_package(Boost REQUIRED COMPONENTS filesystem locale system)
include_directories(${Boost_INCLUDE_DIRS}) include_directories(${Boost_INCLUDE_DIRS})
set(SOURCE_FILES src/main.cpp src/audio_input/WaveFileReader.cpp src/audio_input/WaveFileReader.h src/audio_input/ChannelDownmixer.cpp src/audio_input/ChannelDownmixer.h src/audio_input/AudioStream.h src/audio_input/SampleRateConverter.cpp src/audio_input/SampleRateConverter.h src/audio_input/wave_file_writing.cpp src/audio_input/wave_file_writing.h src/audio_input/io_tools.h src/platform_tools.h src/phone_extraction.cpp src/phone_extraction.h src/Phone.cpp src/Phone.h src/centiseconds.cpp src/centiseconds.h src/tools.cpp src/tools.h) set(SOURCE_FILES src/main.cpp src/audio_input/WaveFileReader.cpp src/audio_input/WaveFileReader.h src/audio_input/ChannelDownmixer.cpp src/audio_input/ChannelDownmixer.h src/audio_input/AudioStream.h src/audio_input/SampleRateConverter.cpp src/audio_input/SampleRateConverter.h src/audio_input/wave_file_writing.cpp src/audio_input/wave_file_writing.h src/audio_input/io_tools.h src/platform_tools.h src/phone_extraction.cpp src/phone_extraction.h src/Phone.cpp src/Phone.h src/centiseconds.cpp src/centiseconds.h src/tools.cpp src/tools.h src/Shape.cpp src/Shape.h src/mouth_animation.cpp src/mouth_animation.h)
if(WIN32) if(WIN32)
set(SOURCE_FILES "${SOURCE_FILES};src/platform_tools_win.cpp") set(SOURCE_FILES "${SOURCE_FILES};src/platform_tools_win.cpp")
else() else()

View File

@ -34,3 +34,6 @@ string phoneToString(Phone phone) {
return (it != phonesByName.right.end()) ? it->second : phoneToString(Phone::Unknown); return (it != phonesByName.right.end()) ? it->second : phoneToString(Phone::Unknown);
} }
std::ostream &operator<<(std::ostream &stream, const Phone phone) {
return stream << phoneToString(phone);
}

View File

@ -74,5 +74,6 @@ Phone stringToPhone(const std::string& s);
std::string phoneToString(Phone phone); std::string phoneToString(Phone phone);
std::ostream& operator <<(std::ostream& stream, const Phone phone);
#endif //LIPSYNC_PHONE_H #endif //LIPSYNC_PHONE_H

10
src/Shape.cpp Normal file
View File

@ -0,0 +1,10 @@
#include "Shape.h"
std::string shapeToString(Shape shape) {
char c = 'A' + static_cast<int>(shape);
return std::string(&c, 1);
}
std::ostream &operator<<(std::ostream &stream, const Shape shape) {
return stream << shapeToString(shape);
}

24
src/Shape.h Normal file
View File

@ -0,0 +1,24 @@
#ifndef LIPSYNC_SHAPE_H
#define LIPSYNC_SHAPE_H
#include <string>
// The classic Hanna-Barbera mouth shapes A-F phus the common supplements G-H
// For reference, see http://sunewatts.dk/lipsync/lipsync/article_02.php
// For visual examples, see https://flic.kr/s/aHsj86KR4J. Their shapes "BMP".."L" map to A..H.
enum class Shape {
A, // Closed mouth (silence, M, B, P)
B, // Clenched teeth (most vowels, m[e]n)
C, // Mouth slightly open (b[ir]d, s[ay], w[i]n...)
D, // Mouth wide open (b[u]t, m[y], sh[ou]ld...)
E, // h[ow]
F, // Pout ([o]ff, sh[ow])
G, // F, V
H // L
};
std::string shapeToString(Shape shape);
std::ostream& operator <<(std::ostream& stream, const Shape shape);
#endif //LIPSYNC_SHAPE_H

View File

@ -1,6 +1,7 @@
#include <iostream> #include <iostream>
#include "audio_input/WaveFileReader.h" #include "audio_input/WaveFileReader.h"
#include "phone_extraction.h" #include "phone_extraction.h"
#include "mouth_animation.h"
#include "platform_tools.h" #include "platform_tools.h"
using std::exception; using std::exception;
@ -42,8 +43,14 @@ int main(int argc, char *argv[]) {
// Detect phones // Detect phones
std::map<centiseconds, Phone> phones = detectPhones(std::move(audioStream)); std::map<centiseconds, Phone> phones = detectPhones(std::move(audioStream));
// Generate mouth shapes
std::map<centiseconds, Shape> shapes = animate(phones);
for (auto &pair : phones) { for (auto &pair : phones) {
std::cout << pair.first << ": " << phoneToString(pair.second) << "\n"; std::cout << pair.first << ": " << pair.second << "\n";
}
for (auto &pair : shapes) {
std::cout << pair.first << ": " << pair.second << "\n";
} }
return 0; return 0;

75
src/mouth_animation.cpp Normal file
View File

@ -0,0 +1,75 @@
#include "mouth_animation.h"
using std::map;
Shape getShape(Phone phone) {
switch (phone) {
case Phone::None:
case Phone::P:
case Phone::B:
case Phone::M:
return Shape::A;
case Phone::Unknown:
case Phone::EH:
case Phone::T:
case Phone::D:
case Phone::K:
case Phone::G:
case Phone::CH:
case Phone::JH:
case Phone::TH:
case Phone::DH:
case Phone::S:
case Phone::Z:
case Phone::SH:
case Phone::ZH:
case Phone::N:
case Phone::NG:
case Phone::R:
return Shape::B;
case Phone::IY:
case Phone::IH:
case Phone::EY:
case Phone::ER:
case Phone::HH:
case Phone::Y:
return Shape::C;
case Phone::UW:
case Phone::UH:
case Phone::AH:
case Phone::AE:
case Phone::AY:
case Phone::OY:
case Phone::W:
return Shape::D;
case Phone::AW:
return Shape::E;
case Phone::AO:
case Phone::AA:
case Phone::OW:
return Shape::F;
case Phone::F:
case Phone::V:
return Shape::G;
case Phone::L:
return Shape::H;
default:
throw std::runtime_error("Unexpected Phone value.");
}
}
map<centiseconds, Shape> animate(const map<centiseconds, Phone> &phones) {
map<centiseconds, Shape> shapes;
for (auto& pair : phones) {
shapes[pair.first] = getShape(pair.second);
}
return shapes;
}

11
src/mouth_animation.h Normal file
View File

@ -0,0 +1,11 @@
#ifndef LIPSYNC_MOUTH_ANIMATION_H
#define LIPSYNC_MOUTH_ANIMATION_H
#include <map>
#include "Phone.h"
#include "centiseconds.h"
#include "Shape.h"
std::map<centiseconds, Shape> animate(const std::map<centiseconds, Phone>& phones);
#endif //LIPSYNC_MOUTH_ANIMATION_H