From 2ef99119b0c88ecdd3e79f714c7360471733bff1 Mon Sep 17 00:00:00 2001 From: Daniel Wolf Date: Fri, 20 Nov 2015 22:20:19 +0100 Subject: [PATCH] Generating mouth shapes using simple lookup table --- .idea/LipSync.iml | 4 +++ CMakeLists.txt | 2 +- src/Phone.cpp | 3 ++ src/Phone.h | 1 + src/Shape.cpp | 10 ++++++ src/Shape.h | 24 +++++++++++++ src/main.cpp | 9 ++++- src/mouth_animation.cpp | 75 +++++++++++++++++++++++++++++++++++++++++ src/mouth_animation.h | 11 ++++++ 9 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 src/Shape.cpp create mode 100644 src/Shape.h create mode 100644 src/mouth_animation.cpp create mode 100644 src/mouth_animation.h diff --git a/.idea/LipSync.iml b/.idea/LipSync.iml index 540160f..a1cde36 100644 --- a/.idea/LipSync.iml +++ b/.idea/LipSync.iml @@ -139,12 +139,16 @@ + + + + diff --git a/CMakeLists.txt b/CMakeLists.txt index dbc9835..7d9eca9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ set(Boost_USE_STATIC_RUNTIME ON) # Use static C++ runtime find_package(Boost REQUIRED COMPONENTS filesystem locale system) include_directories(${Boost_INCLUDE_DIRS}) -set(SOURCE_FILES src/main.cpp src/audio_input/WaveFileReader.cpp src/audio_input/WaveFileReader.h src/audio_input/ChannelDownmixer.cpp src/audio_input/ChannelDownmixer.h src/audio_input/AudioStream.h src/audio_input/SampleRateConverter.cpp src/audio_input/SampleRateConverter.h src/audio_input/wave_file_writing.cpp src/audio_input/wave_file_writing.h src/audio_input/io_tools.h src/platform_tools.h src/phone_extraction.cpp src/phone_extraction.h src/Phone.cpp src/Phone.h src/centiseconds.cpp src/centiseconds.h src/tools.cpp src/tools.h) +set(SOURCE_FILES src/main.cpp src/audio_input/WaveFileReader.cpp src/audio_input/WaveFileReader.h src/audio_input/ChannelDownmixer.cpp src/audio_input/ChannelDownmixer.h src/audio_input/AudioStream.h src/audio_input/SampleRateConverter.cpp src/audio_input/SampleRateConverter.h src/audio_input/wave_file_writing.cpp src/audio_input/wave_file_writing.h src/audio_input/io_tools.h src/platform_tools.h src/phone_extraction.cpp src/phone_extraction.h src/Phone.cpp src/Phone.h src/centiseconds.cpp src/centiseconds.h src/tools.cpp src/tools.h src/Shape.cpp src/Shape.h src/mouth_animation.cpp src/mouth_animation.h) if(WIN32) set(SOURCE_FILES "${SOURCE_FILES};src/platform_tools_win.cpp") else() diff --git a/src/Phone.cpp b/src/Phone.cpp index f601e39..82d7792 100644 --- a/src/Phone.cpp +++ b/src/Phone.cpp @@ -34,3 +34,6 @@ string phoneToString(Phone phone) { return (it != phonesByName.right.end()) ? it->second : phoneToString(Phone::Unknown); } +std::ostream &operator<<(std::ostream &stream, const Phone phone) { + return stream << phoneToString(phone); +} diff --git a/src/Phone.h b/src/Phone.h index 4928513..7b0eba2 100644 --- a/src/Phone.h +++ b/src/Phone.h @@ -74,5 +74,6 @@ Phone stringToPhone(const std::string& s); std::string phoneToString(Phone phone); +std::ostream& operator <<(std::ostream& stream, const Phone phone); #endif //LIPSYNC_PHONE_H diff --git a/src/Shape.cpp b/src/Shape.cpp new file mode 100644 index 0000000..ed4f85b --- /dev/null +++ b/src/Shape.cpp @@ -0,0 +1,10 @@ +#include "Shape.h" + +std::string shapeToString(Shape shape) { + char c = 'A' + static_cast(shape); + return std::string(&c, 1); +} + +std::ostream &operator<<(std::ostream &stream, const Shape shape) { + return stream << shapeToString(shape); +} diff --git a/src/Shape.h b/src/Shape.h new file mode 100644 index 0000000..d1018a7 --- /dev/null +++ b/src/Shape.h @@ -0,0 +1,24 @@ +#ifndef LIPSYNC_SHAPE_H +#define LIPSYNC_SHAPE_H + +#include + +// The classic Hanna-Barbera mouth shapes A-F phus the common supplements G-H +// For reference, see http://sunewatts.dk/lipsync/lipsync/article_02.php +// For visual examples, see https://flic.kr/s/aHsj86KR4J. Their shapes "BMP".."L" map to A..H. +enum class Shape { + A, // Closed mouth (silence, M, B, P) + B, // Clenched teeth (most vowels, m[e]n) + C, // Mouth slightly open (b[ir]d, s[ay], w[i]n...) + D, // Mouth wide open (b[u]t, m[y], sh[ou]ld...) + E, // h[ow] + F, // Pout ([o]ff, sh[ow]) + G, // F, V + H // L +}; + +std::string shapeToString(Shape shape); + +std::ostream& operator <<(std::ostream& stream, const Shape shape); + +#endif //LIPSYNC_SHAPE_H diff --git a/src/main.cpp b/src/main.cpp index 33a1db1..2a84772 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,6 +1,7 @@ #include #include "audio_input/WaveFileReader.h" #include "phone_extraction.h" +#include "mouth_animation.h" #include "platform_tools.h" using std::exception; @@ -42,8 +43,14 @@ int main(int argc, char *argv[]) { // Detect phones std::map phones = detectPhones(std::move(audioStream)); + // Generate mouth shapes + std::map shapes = animate(phones); + for (auto &pair : phones) { - std::cout << pair.first << ": " << phoneToString(pair.second) << "\n"; + std::cout << pair.first << ": " << pair.second << "\n"; + } + for (auto &pair : shapes) { + std::cout << pair.first << ": " << pair.second << "\n"; } return 0; diff --git a/src/mouth_animation.cpp b/src/mouth_animation.cpp new file mode 100644 index 0000000..dd6128e --- /dev/null +++ b/src/mouth_animation.cpp @@ -0,0 +1,75 @@ +#include "mouth_animation.h" + +using std::map; + +Shape getShape(Phone phone) { + switch (phone) { + case Phone::None: + case Phone::P: + case Phone::B: + case Phone::M: + return Shape::A; + + case Phone::Unknown: + case Phone::EH: + case Phone::T: + case Phone::D: + case Phone::K: + case Phone::G: + case Phone::CH: + case Phone::JH: + case Phone::TH: + case Phone::DH: + case Phone::S: + case Phone::Z: + case Phone::SH: + case Phone::ZH: + case Phone::N: + case Phone::NG: + case Phone::R: + return Shape::B; + + case Phone::IY: + case Phone::IH: + case Phone::EY: + case Phone::ER: + case Phone::HH: + case Phone::Y: + return Shape::C; + + case Phone::UW: + case Phone::UH: + case Phone::AH: + case Phone::AE: + case Phone::AY: + case Phone::OY: + case Phone::W: + return Shape::D; + + case Phone::AW: + return Shape::E; + + case Phone::AO: + case Phone::AA: + case Phone::OW: + return Shape::F; + + case Phone::F: + case Phone::V: + return Shape::G; + + case Phone::L: + return Shape::H; + + default: + throw std::runtime_error("Unexpected Phone value."); + } +} + +map animate(const map &phones) { + map shapes; + for (auto& pair : phones) { + shapes[pair.first] = getShape(pair.second); + } + return shapes; +} diff --git a/src/mouth_animation.h b/src/mouth_animation.h new file mode 100644 index 0000000..489d2af --- /dev/null +++ b/src/mouth_animation.h @@ -0,0 +1,11 @@ +#ifndef LIPSYNC_MOUTH_ANIMATION_H +#define LIPSYNC_MOUTH_ANIMATION_H + +#include +#include "Phone.h" +#include "centiseconds.h" +#include "Shape.h" + +std::map animate(const std::map& phones); + +#endif //LIPSYNC_MOUTH_ANIMATION_H