rhubarb-lip-sync/src/mouthAnimation.cpp

159 lines
5.9 KiB
C++
Raw Normal View History

#include "mouthAnimation.h"
2016-03-01 20:57:05 +00:00
#include "logging.h"
#include <unordered_set>
#include <unordered_map>
#include <array>
using std::map;
using std::unordered_set;
using std::unordered_map;
using std::vector;
using boost::optional;
using AnimationResult = Timeline<Shape>;
AnimationResult animateFixedSound(Shape shape, centiseconds duration) {
return AnimationResult{ {0cs, duration, shape} };
}
// Diphtong vowels
AnimationResult animateDiphtong(Shape first, Shape second, centiseconds duration) {
return AnimationResult{
{ 0cs, duration, first },
{ duration / 2, duration, second }
};
}
// P, B
AnimationResult animateBilabialStop(centiseconds duration, centiseconds leftPhoneDuration, optional<Shape> rightShape) {
Shape openShape = rightShape.value_or(Shape::B);
if (openShape == Shape::A) {
openShape = Shape::B;
}
centiseconds closedShapeDuration = leftPhoneDuration / 2;
if (closedShapeDuration.count() < 4) closedShapeDuration = 4cs;
if (closedShapeDuration.count() > 16) closedShapeDuration = 16cs;
return AnimationResult{
{ -closedShapeDuration, 0cs, Shape::A },
{ 0cs, duration, openShape }
};
}
// Sounds with no fixed mouth position.
// Mapping specifies the shape to use for every right shape.
AnimationResult animateFlexibleSound(std::array<Shape, 7> mapping, centiseconds duration, optional<Shape> rightShape) {
constexpr int mapSize = std::tuple_size<decltype(mapping)>::value;
static_assert(static_cast<int>(Shape::EndSentinel) == mapSize, "Shape definition has changed.");
Shape right = rightShape.value_or(Shape::A);
Shape shape = mapping[static_cast<int>(right)];
return AnimationResult{ { 0cs, duration, shape } };
}
AnimationResult animate(optional<Phone> phone, centiseconds duration, centiseconds leftPhoneDuration, optional<Shape> rightShape) {
constexpr Shape A = Shape::A;
constexpr Shape B = Shape::B;
constexpr Shape C = Shape::C;
constexpr Shape D = Shape::D;
constexpr Shape E = Shape::E;
constexpr Shape F = Shape::F;
constexpr Shape G = Shape::G;
if (!phone) {
return animateFixedSound(A, duration);
}
switch (*phone) {
case Phone::Unknown: return animateFixedSound(B, duration);
case Phone::AO: return animateFixedSound(E, duration);
case Phone::AA: return animateFixedSound(D, duration);
case Phone::IY: return animateFixedSound(B, duration);
case Phone::UW: return animateFixedSound(F, duration);
case Phone::EH: return animateFixedSound(C, duration);
case Phone::IH: return animateFixedSound(B, duration);
case Phone::UH: return animateFixedSound(E, duration);
case Phone::AH: return animateFixedSound(C, duration);
case Phone::AE: return animateFixedSound(D, duration);
case Phone::EY: return animateDiphtong(C, B, duration);
case Phone::AY: return animateDiphtong(D, B, duration);
case Phone::OW: return animateDiphtong(E, F, duration);
case Phone::AW: return animateDiphtong(D, F, duration);
case Phone::OY: return animateDiphtong(E, B, duration);
case Phone::ER: return animateFixedSound(E, duration);
case Phone::P:
case Phone::B: return animateBilabialStop(duration, leftPhoneDuration, rightShape);
case Phone::T:
case Phone::D:
case Phone::K: return animateFlexibleSound({ B, B, B, B, B, F, B }, duration, rightShape);
case Phone::G: return animateFlexibleSound({ B, B, C, C, E, F, B }, duration, rightShape);
case Phone::CH:
case Phone::JH: return animateFlexibleSound({ B, B, B, B, B, F, B }, duration, rightShape);
case Phone::F:
case Phone::V: return animateFixedSound(G, duration);
case Phone::TH:
case Phone::DH:
case Phone::S:
case Phone::Z:
case Phone::SH:
case Phone::ZH: return animateFlexibleSound({ B, B, B, B, B, F, B }, duration, rightShape);
case Phone::HH: return animateFlexibleSound({ B, B, C, D, E, F, B }, duration, rightShape);
case Phone::M: return animateFixedSound(A, duration);
case Phone::N: return animateFlexibleSound({ B, B, C, C, C, F, B }, duration, rightShape);
case Phone::NG: return animateFlexibleSound({ B, B, C, D, E, F, B }, duration, rightShape);
case Phone::L: return animateFlexibleSound({ C, C, C, D, E, F, C }, duration, rightShape);
case Phone::R: return animateFlexibleSound({ B, B, B, B, B, F, B }, duration, rightShape);
case Phone::Y: return animateFixedSound(B, duration);
case Phone::W: return animateFixedSound(F, duration);
default:
throw std::invalid_argument("Unexpected phone.");
}
}
ContinuousTimeline<Shape> animate(const BoundedTimeline<Phone> &phones) {
// Convert phones to continuous timeline so that silences show up when iterating
ContinuousTimeline<optional<Phone>> continuousPhones(phones.getRange(), boost::none);
for (const auto& timedPhone : phones) {
continuousPhones.set(timedPhone.getTimeRange(), timedPhone.getValue());
}
ContinuousTimeline<Shape> shapes(phones.getRange(), Shape::A);
// Iterate phones in *reverse* order so we can access the right-hand result
optional<Shape> lastShape;
centiseconds cutoff = shapes.getRange().getEnd();
for (auto it = continuousPhones.rbegin(); it != continuousPhones.rend(); ++it) {
// Animate one phone
optional<Phone> phone = it->getValue();
centiseconds duration = it->getTimeRange().getLength();
bool hasLeftPhone = std::next(it) != continuousPhones.rend() && std::next(it)->getEnd() == it->getStart();
centiseconds leftPhoneDuration = hasLeftPhone
? std::next(it)->getTimeRange().getLength()
: 0cs;
Timeline<Shape> result = animate(phone, duration, leftPhoneDuration, lastShape);
// Result timing is relative to phone. Make absolute.
result.shift(it->getStart());
// New shapes must not overwrite existing shapes
result.clear(cutoff, shapes.getRange().getEnd());
// Copy to target timeline
for (const auto& timedShape : result) {
shapes.set(timedShape);
}
lastShape = result.empty() ? optional<Shape>() : result.begin()->getValue();
if (!result.empty()) {
cutoff = result.begin()->getStart();
}
}
for (const auto& timedShape : shapes) {
logging::logTimedEvent("shape", timedShape);
2016-03-01 20:57:05 +00:00
}
return shapes;
}