From 2a5ed95698392f358addc2963c7bcd8814768f68 Mon Sep 17 00:00:00 2001 From: Daniel Wolf Date: Sun, 26 Jun 2016 20:11:02 +0200 Subject: [PATCH] Improved animation quality through new algorithm Using "lazy" ruleset instead of 1:1 mapping from phones --- src/Shape.h | 3 +- src/mouthAnimation.cpp | 181 +++++++++++++++++++++++++++++------------ 2 files changed, 129 insertions(+), 55 deletions(-) diff --git a/src/Shape.h b/src/Shape.h index 9932a7e..331be2d 100644 --- a/src/Shape.h +++ b/src/Shape.h @@ -12,7 +12,8 @@ enum class Shape { D, // Mouth wide open (b[u]t, m[y], sh[ou]ld...) E, // h[ow] F, // Pout ([o]ff, sh[ow]) - G // F, V + G, // F, V + EndSentinel }; class ShapeConverter : public EnumConverter { diff --git a/src/mouthAnimation.cpp b/src/mouthAnimation.cpp index 23b364e..cdf9291 100644 --- a/src/mouthAnimation.cpp +++ b/src/mouthAnimation.cpp @@ -1,74 +1,147 @@ #include "mouthAnimation.h" #include "logging.h" +#include +#include +#include using std::map; +using std::unordered_set; +using std::unordered_map; +using std::vector; +using boost::optional; -Shape getShape(Phone phone) { - switch (phone) { - case Phone::P: - case Phone::B: - case Phone::M: - return Shape::A; +using AnimationResult = Timeline; - case Phone::Unknown: - case Phone::IY: - case Phone::T: - case Phone::D: - case Phone::K: - case Phone::G: - case Phone::CH: - case Phone::JH: - case Phone::TH: - case Phone::DH: - case Phone::S: - case Phone::Z: - case Phone::SH: - case Phone::ZH: - case Phone::N: - case Phone::NG: - case Phone::R: - case Phone::Y: - return Shape::B; +AnimationResult animateFixedSound(Shape shape, centiseconds duration) { + return AnimationResult{ {centiseconds::zero(), duration, shape} }; +} - case Phone::EH: - case Phone::IH: - case Phone::AH: - case Phone::EY: - case Phone::HH: - case Phone::L: - return Shape::C; +// Diphtong vowels +AnimationResult animateDiphtong(Shape first, Shape second, centiseconds duration) { + return AnimationResult{ + { centiseconds::zero(), duration, first }, + { duration / 2, duration, second } + }; +} - case Phone::AA: - case Phone::AE: - case Phone::AY: - case Phone::AW: - return Shape::D; +// P, B +AnimationResult animateBilabialStop(centiseconds duration, centiseconds leftDuration, optional rightShape) { + Shape openShape = rightShape.value_or(Shape::B); + if (openShape == Shape::A) { + openShape = Shape::B; + } + return AnimationResult{ + { -leftDuration / 4, centiseconds::zero(), Shape::A }, + { centiseconds::zero(), duration, openShape } + }; +} - case Phone::AO: - case Phone::UH: - case Phone::OW: - case Phone::ER: - return Shape::E; +// Sounds with no fixed mouth position. +// Mapping specifies the shape to use for every right shape. +AnimationResult animateFlexibleSound(std::array mapping, centiseconds duration, optional rightShape) { + constexpr int mapSize = std::tuple_size::value; + static_assert(static_cast(Shape::EndSentinel) == mapSize, "Shape definition has changed."); - case Phone::UW: - case Phone::OY: - case Phone::W: - return Shape::F; + Shape right = rightShape.value_or(Shape::A); + Shape shape = mapping[static_cast(right)]; + return AnimationResult{ { centiseconds::zero(), duration, shape } }; +} - case Phone::F: - case Phone::V: - return Shape::G; +AnimationResult animate(optional phone, centiseconds duration, centiseconds leftDuration, optional rightShape) { + constexpr Shape A = Shape::A; + constexpr Shape B = Shape::B; + constexpr Shape C = Shape::C; + constexpr Shape D = Shape::D; + constexpr Shape E = Shape::E; + constexpr Shape F = Shape::F; + constexpr Shape G = Shape::G; - default: - throw std::runtime_error("Unexpected Phone value."); + if (!phone) { + return animateFixedSound(A, duration); + } + + switch (*phone) { + case Phone::Unknown: return animateFixedSound(B, duration); + case Phone::AO: return animateFixedSound(E, duration); + case Phone::AA: return animateFixedSound(D, duration); + case Phone::IY: return animateFixedSound(B, duration); + case Phone::UW: return animateFixedSound(F, duration); + case Phone::EH: return animateFixedSound(C, duration); + case Phone::IH: return animateFixedSound(B, duration); + case Phone::UH: return animateFixedSound(E, duration); + case Phone::AH: return animateFixedSound(C, duration); + case Phone::AE: return animateFixedSound(D, duration); + case Phone::EY: return animateDiphtong(C, B, duration); + case Phone::AY: return animateDiphtong(D, B, duration); + case Phone::OW: return animateDiphtong(E, F, duration); + case Phone::AW: return animateDiphtong(D, F, duration); + case Phone::OY: return animateDiphtong(E, B, duration); + case Phone::ER: return animateFixedSound(E, duration); + case Phone::P: + case Phone::B: return animateBilabialStop(duration, leftDuration, rightShape); + case Phone::T: + case Phone::D: + case Phone::K: return animateFlexibleSound({ B, B, B, B, B, F, B }, duration, rightShape); + case Phone::G: return animateFlexibleSound({ B, B, C, C, E, F, B }, duration, rightShape); + case Phone::CH: + case Phone::JH: return animateFlexibleSound({ B, B, B, B, B, F, B }, duration, rightShape); + case Phone::F: + case Phone::V: return animateFixedSound(G, duration); + case Phone::TH: + case Phone::DH: + case Phone::S: + case Phone::Z: + case Phone::SH: + case Phone::ZH: return animateFlexibleSound({ B, B, B, B, B, F, B }, duration, rightShape); + case Phone::HH: return animateFlexibleSound({ B, B, C, D, E, F, B }, duration, rightShape); + case Phone::M: return animateFixedSound(A, duration); + case Phone::N: return animateFlexibleSound({ B, B, C, C, C, F, B }, duration, rightShape); + case Phone::NG: + case Phone::L: return animateFlexibleSound({ B, B, C, D, E, F, B }, duration, rightShape); + case Phone::R: return animateFlexibleSound({ B, B, B, B, B, F, B }, duration, rightShape); + case Phone::Y: return animateFixedSound(B, duration); + case Phone::W: return animateFixedSound(F, duration); + default: + throw std::invalid_argument("Unexpected phone."); } } ContinuousTimeline animate(const BoundedTimeline &phones) { - ContinuousTimeline shapes(phones.getRange(), Shape::A); + // Convert phones to continuous timeline so that silences show up when iterating + ContinuousTimeline> continuousPhones(phones.getRange(), boost::none); for (const auto& timedPhone : phones) { - Timed timedShape(timedPhone.getTimeRange(), getShape(timedPhone.getValue())); - shapes.set(timedShape); + continuousPhones.set(timedPhone.getTimeRange(), timedPhone.getValue()); + } + + ContinuousTimeline shapes(phones.getRange(), Shape::A); + + // Iterate phones in *reverse* order so we can access the right-hand result + optional lastShape; + centiseconds cutoff = shapes.getRange().getEnd(); + for (auto it = continuousPhones.rbegin(); it != continuousPhones.rend(); ++it) { + // Animate one phone + optional phone = it->getValue(); + centiseconds duration = it->getTimeRange().getLength(); + centiseconds leftDuration = std::next(it) != continuousPhones.rend() + ? std::next(it)->getTimeRange().getLength() + : centiseconds::zero(); + Timeline result = animate(phone, duration, leftDuration, lastShape); + + // Result timing is relative to phone. Make absolute. + result.shift(it->getStart()); + + // New shapes must not overwrite existing shapes + result.clear(cutoff, shapes.getRange().getEnd()); + + // Copy to target timeline + for (const auto& timedShape : result) { + shapes.set(timedShape); + } + + lastShape = result.empty() ? optional() : result.begin()->getValue(); + if (!result.empty()) { + cutoff = result.begin()->getStart(); + } } for (const auto& timedShape : shapes) {