2015-12-29 15:26:01 +00:00
|
|
|
#include "mouthAnimation.h"
|
2016-03-01 20:57:05 +00:00
|
|
|
#include "logging.h"
|
2016-06-26 18:11:02 +00:00
|
|
|
#include <unordered_set>
|
|
|
|
#include <unordered_map>
|
2016-07-31 19:42:37 +00:00
|
|
|
#include <boost/algorithm/clamp.hpp>
|
|
|
|
#include "Viseme.h"
|
2015-11-20 21:20:19 +00:00
|
|
|
|
|
|
|
using std::map;
|
2016-06-26 18:11:02 +00:00
|
|
|
using std::unordered_set;
|
|
|
|
using std::unordered_map;
|
|
|
|
using std::vector;
|
|
|
|
using boost::optional;
|
2016-07-31 19:42:37 +00:00
|
|
|
using std::chrono::duration_cast;
|
|
|
|
using boost::algorithm::clamp;
|
2016-08-02 20:02:59 +00:00
|
|
|
using std::pair;
|
|
|
|
using std::tuple;
|
|
|
|
|
|
|
|
constexpr Shape A = Shape::A;
|
|
|
|
constexpr Shape B = Shape::B;
|
|
|
|
constexpr Shape C = Shape::C;
|
|
|
|
constexpr Shape D = Shape::D;
|
|
|
|
constexpr Shape E = Shape::E;
|
|
|
|
constexpr Shape F = Shape::F;
|
|
|
|
constexpr Shape G = Shape::G;
|
|
|
|
constexpr Shape H = Shape::H;
|
|
|
|
constexpr Shape X = Shape::X;
|
2015-11-20 21:20:19 +00:00
|
|
|
|
2016-07-31 19:42:37 +00:00
|
|
|
Timeline<Viseme> animate(optional<Phone> phone, centiseconds duration, centiseconds previousPhoneDuration) {
|
|
|
|
auto single = [&](Viseme viseme) {
|
|
|
|
return Timeline<Viseme>{
|
|
|
|
{ 0cs, duration, viseme }
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
auto diphtong = [&](Viseme first, Viseme second) {
|
|
|
|
centiseconds firstDuration = duration_cast<centiseconds>(duration * 0.6);
|
|
|
|
return Timeline<Viseme>{
|
|
|
|
{ 0cs, firstDuration, first },
|
|
|
|
{ firstDuration, duration, second }
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
auto bilabialStop = [&]() {
|
|
|
|
centiseconds closedDuration = clamp(previousPhoneDuration / 2, 4cs, 16cs);
|
|
|
|
return Timeline<Viseme>{
|
|
|
|
{ -closedDuration, 0cs, { A } },
|
|
|
|
{ 0cs, duration, {{ B, C, D, E, F }} }
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
if (!phone) return single({ X });
|
2016-06-26 18:11:02 +00:00
|
|
|
|
|
|
|
switch (*phone) {
|
2016-07-31 19:42:37 +00:00
|
|
|
case Phone::Unknown: return single({ B });
|
|
|
|
case Phone::AO: return single({ E });
|
|
|
|
case Phone::AA: return single({ D });
|
|
|
|
case Phone::IY: return single({ B });
|
|
|
|
case Phone::UW: return single({ F });
|
|
|
|
case Phone::EH: return single({ C });
|
|
|
|
case Phone::IH: return single({ B });
|
|
|
|
case Phone::UH: return single({ E });
|
|
|
|
case Phone::AH: return single({ C });
|
|
|
|
case Phone::AE: return single({ D });
|
|
|
|
case Phone::EY: return diphtong({ C }, { B });
|
|
|
|
case Phone::AY: return diphtong({ D }, { B });
|
|
|
|
case Phone::OW: return diphtong({ E }, { F });
|
|
|
|
case Phone::AW: return diphtong({ D }, { F });
|
|
|
|
case Phone::OY: return diphtong({ E }, { B });
|
|
|
|
case Phone::ER: return single({ E });
|
2016-06-26 18:11:02 +00:00
|
|
|
case Phone::P:
|
2016-07-31 19:42:37 +00:00
|
|
|
case Phone::B: return bilabialStop();
|
2016-06-26 18:11:02 +00:00
|
|
|
case Phone::T:
|
|
|
|
case Phone::D:
|
2016-07-31 19:42:37 +00:00
|
|
|
case Phone::K: return single({ { B, B, B, B, F } });
|
|
|
|
case Phone::G: return single({ { B, C, C, E, F } });
|
2016-06-26 18:11:02 +00:00
|
|
|
case Phone::CH:
|
2016-07-31 19:42:37 +00:00
|
|
|
case Phone::JH: return single({ { B, B, B, B, F } });
|
2016-06-26 18:11:02 +00:00
|
|
|
case Phone::F:
|
2016-07-31 19:42:37 +00:00
|
|
|
case Phone::V: return single({ G });
|
2016-06-26 18:11:02 +00:00
|
|
|
case Phone::TH:
|
|
|
|
case Phone::DH:
|
|
|
|
case Phone::S:
|
|
|
|
case Phone::Z:
|
|
|
|
case Phone::SH:
|
2016-07-31 19:42:37 +00:00
|
|
|
case Phone::ZH: return single({ { B, B, B, B, F } });
|
|
|
|
case Phone::HH: return single({ { B, C, D, E, F } });
|
|
|
|
case Phone::M: return single({ A });
|
|
|
|
case Phone::N: return single({ { B, C, C, C, F } });
|
|
|
|
case Phone::NG: return single({ { B, C, D, E, F } });
|
|
|
|
case Phone::L: return single({ { H, H, H, E, F } });
|
|
|
|
case Phone::R: return single({ { B, B, B, B, F } });
|
|
|
|
case Phone::Y: return single({ B });
|
|
|
|
case Phone::W: return single({ F });
|
2016-06-26 18:11:02 +00:00
|
|
|
default:
|
|
|
|
throw std::invalid_argument("Unexpected phone.");
|
2015-11-20 21:20:19 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-08-02 20:02:59 +00:00
|
|
|
enum class TweenTiming {
|
|
|
|
Early,
|
|
|
|
Centered,
|
|
|
|
Late
|
|
|
|
};
|
|
|
|
|
|
|
|
optional<pair<Shape, TweenTiming>> getTween(Shape first, Shape second) {
|
|
|
|
static const map<pair<Shape, Shape>, pair<Shape, TweenTiming>> lookup {
|
|
|
|
{ { A, D }, { C, TweenTiming::Late } }, { { D, A },{ C, TweenTiming::Early } },
|
|
|
|
{ { B, D }, { C, TweenTiming::Centered } }, { { D, B },{ C, TweenTiming::Centered } },
|
|
|
|
{ { G, D }, { C, TweenTiming::Late } }, { { D, G },{ C, TweenTiming::Early } },
|
|
|
|
{ { X, D }, { C, TweenTiming::Early } }, { { D, X },{ C, TweenTiming::Late } },
|
|
|
|
{ { C, F }, { E, TweenTiming::Centered } }, { { F, C },{ E, TweenTiming::Centered } },
|
|
|
|
{ { D, F }, { E, TweenTiming::Centered } }, { { F, D },{ E, TweenTiming::Centered } },
|
|
|
|
{ { H, F }, { E, TweenTiming::Late } }, { { F, H },{ E, TweenTiming::Early } },
|
|
|
|
};
|
|
|
|
auto it = lookup.find({ first, second });
|
|
|
|
return it != lookup.end() ? it->second : optional<pair<Shape, TweenTiming>>();
|
|
|
|
}
|
|
|
|
|
2016-05-02 18:31:59 +00:00
|
|
|
ContinuousTimeline<Shape> animate(const BoundedTimeline<Phone> &phones) {
|
2016-07-31 19:42:37 +00:00
|
|
|
// Convert phones to continuous timeline so that silences aren't skipped when iterating
|
2016-06-26 18:11:02 +00:00
|
|
|
ContinuousTimeline<optional<Phone>> continuousPhones(phones.getRange(), boost::none);
|
2016-05-02 18:31:59 +00:00
|
|
|
for (const auto& timedPhone : phones) {
|
2016-06-26 18:11:02 +00:00
|
|
|
continuousPhones.set(timedPhone.getTimeRange(), timedPhone.getValue());
|
|
|
|
}
|
|
|
|
|
2016-07-31 19:42:37 +00:00
|
|
|
// Create timeline of visemes
|
2016-08-02 20:02:59 +00:00
|
|
|
ContinuousTimeline<Viseme> visemes(phones.getRange(), { X });
|
2016-07-31 19:42:37 +00:00
|
|
|
centiseconds previousPhoneDuration = 0cs;
|
|
|
|
for (const auto& timedPhone : continuousPhones) {
|
2016-06-26 18:11:02 +00:00
|
|
|
// Animate one phone
|
2016-07-31 19:42:37 +00:00
|
|
|
optional<Phone> phone = timedPhone.getValue();
|
|
|
|
centiseconds duration = timedPhone.getTimeRange().getLength();
|
|
|
|
Timeline<Viseme> phoneVisemes = animate(phone, duration, previousPhoneDuration);
|
2016-06-26 18:11:02 +00:00
|
|
|
|
|
|
|
// Result timing is relative to phone. Make absolute.
|
2016-07-31 19:42:37 +00:00
|
|
|
phoneVisemes.shift(timedPhone.getStart());
|
2016-06-26 18:11:02 +00:00
|
|
|
|
2016-07-31 19:42:37 +00:00
|
|
|
// Copy to viseme timeline
|
|
|
|
for (const auto& timedViseme : phoneVisemes) {
|
|
|
|
visemes.set(timedViseme);
|
2016-06-26 18:11:02 +00:00
|
|
|
}
|
|
|
|
|
2016-07-31 19:42:37 +00:00
|
|
|
previousPhoneDuration = duration;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create timeline of shapes.
|
|
|
|
// Iterate visemes in *reverse* order so we always know what shape will follow.
|
2016-08-02 20:02:59 +00:00
|
|
|
ContinuousTimeline<Shape> shapes(phones.getRange(), X);
|
|
|
|
Shape lastShape = X;
|
2016-07-31 19:42:37 +00:00
|
|
|
for (auto it = visemes.rbegin(); it != visemes.rend(); ++it) {
|
|
|
|
Viseme viseme = it->getValue();
|
|
|
|
|
|
|
|
// Convert viseme to phone
|
|
|
|
Shape shape = viseme.getShape(it->getTimeRange().getLength(), lastShape);
|
|
|
|
|
|
|
|
shapes.set(it->getTimeRange(), shape);
|
2016-05-02 18:31:59 +00:00
|
|
|
}
|
|
|
|
|
2016-08-02 20:02:59 +00:00
|
|
|
// Create inbetweens for smoother animation
|
|
|
|
centiseconds minTweenDuration = 4cs;
|
|
|
|
centiseconds maxTweenDuration = 10cs;
|
|
|
|
Timeline<Shape> tweens;
|
|
|
|
for (auto first = shapes.begin(), second = std::next(shapes.begin());
|
|
|
|
first != shapes.end() && second != shapes.end();
|
|
|
|
++first, ++second)
|
|
|
|
{
|
|
|
|
auto pair = getTween(first->getValue(), second->getValue());
|
|
|
|
if (!pair) continue;
|
|
|
|
|
|
|
|
Shape tweenShape;
|
|
|
|
TweenTiming tweenTiming;
|
|
|
|
std::tie(tweenShape, tweenTiming) = *pair;
|
|
|
|
TimeRange firstTimeRange = first->getTimeRange();
|
|
|
|
TimeRange secondTimeRange = second->getTimeRange();
|
|
|
|
|
|
|
|
centiseconds tweenStart, tweenDuration;
|
|
|
|
switch (tweenTiming) {
|
|
|
|
case TweenTiming::Early: {
|
|
|
|
tweenDuration = std::min(firstTimeRange.getLength() / 3, maxTweenDuration);
|
|
|
|
tweenStart = firstTimeRange.getEnd() - tweenDuration;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case TweenTiming::Centered: {
|
|
|
|
tweenDuration = std::min({ firstTimeRange.getLength() / 3, secondTimeRange.getLength() / 3, maxTweenDuration });
|
|
|
|
tweenStart = firstTimeRange.getEnd() - tweenDuration / 2;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case TweenTiming::Late: {
|
|
|
|
tweenDuration = std::min(secondTimeRange.getLength() / 3, maxTweenDuration);
|
|
|
|
tweenStart = secondTimeRange.getStart();
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (tweenDuration < minTweenDuration) continue;
|
|
|
|
|
|
|
|
tweens.set(tweenStart, tweenStart + tweenDuration, tweenShape);
|
|
|
|
}
|
|
|
|
for (const auto& tween : tweens) {
|
|
|
|
shapes.set(tween);
|
|
|
|
}
|
|
|
|
|
2016-05-02 18:31:59 +00:00
|
|
|
for (const auto& timedShape : shapes) {
|
2016-04-13 20:37:39 +00:00
|
|
|
logging::logTimedEvent("shape", timedShape);
|
2016-03-01 20:57:05 +00:00
|
|
|
}
|
|
|
|
|
2015-11-20 21:20:19 +00:00
|
|
|
return shapes;
|
|
|
|
}
|