Re-written animation code

* Still uses (almost) the same rules, but more powerful underlying concept
* Re-introduced shape H for "L" sounds
* Introduced shape X for idle position

CMakeLists.txt

@@ -193,6 +193,7 @@ set(SOURCE_FILES
 	src/Shape.cpp src/Shape.h
 	src/centiseconds.cpp src/centiseconds.h
 	src/EnumConverter.h
+	src/Viseme.cpp src/Viseme.h
 	src/mouthAnimation.cpp src/mouthAnimation.h
 	src/phoneExtraction.cpp src/phoneExtraction.h
 	src/platformTools.cpp src/platformTools.h

src/Shape.cpp

@@ -19,7 +19,9 @@ EnumConverter<Shape>::member_data ShapeConverter::getMemberData() {
 		{ Shape::D, "D" },
 		{ Shape::E, "E" },
 		{ Shape::F, "F" },
-		{ Shape::G, "G" }
+		{ Shape::G, "G" },
+		{ Shape::H, "H" },
+		{ Shape::X, "X" }
 	};
 }
 

src/Shape.h

@@ -13,6 +13,8 @@ enum class Shape {
 	E,	// h[ow]
 	F,	// Pout ([o]ff, sh[ow])
 	G,	// F, V
+	H,	// L
+	X,	// Idle
 	EndSentinel
 };
 

src/Viseme.cpp

@@ -0,0 +1,52 @@
+#include "Viseme.h"
+
+VisemeOption::VisemeOption(Shape shape) :
+	VisemeOption(shape, shape, shape, shape, shape)
+{}
+
+VisemeOption::VisemeOption(Shape nearB, Shape nearC, Shape nearD, Shape nearE, Shape nearF) :
+	VisemeOption(nearB, nearB, nearC, nearD, nearE, nearF, nearB, nearC, nearB)
+{}
+
+VisemeOption::VisemeOption(Shape nearA, Shape nearB, Shape nearC, Shape nearD, Shape nearE, Shape nearF, Shape nearG, Shape nearH, Shape nearX) :
+	shapes{ nearA, nearB, nearC, nearD, nearE, nearF, nearG, nearH, nearX }
+{
+	static_assert(static_cast<int>(Shape::EndSentinel) == 9, "Shape definition has changed.");
+}
+
+Shape VisemeOption::getShape(Shape context) const {
+	return shapes.at(static_cast<int>(context));
+}
+
+bool VisemeOption::operator==(const VisemeOption& rhs) const {
+	return shapes == rhs.shapes;
+}
+
+bool VisemeOption::operator!=(const VisemeOption& rhs) const {
+	return !operator==(rhs);
+}
+
+Viseme::Viseme(const VisemeOption& option) :
+	options{ { 0cs, option } }
+{}
+
+Viseme::Viseme(const VisemeOption& option1, centiseconds threshold, const VisemeOption& option2) :
+	options{ { 0cs, option1 }, { threshold, option2 } }
+{}
+
+Viseme::Viseme(const VisemeOption& option1, centiseconds threshold1, const VisemeOption& option2, centiseconds threshold2, const VisemeOption& option3) :
+	options{ { 0cs, option1 },{ threshold1, option2 }, { threshold2, option3 } }
+{}
+
+Shape Viseme::getShape(centiseconds duration, Shape context) const {
+	VisemeOption option = std::prev(options.upper_bound(duration))->second;
+	return option.getShape(context);
+}
+
+bool Viseme::operator==(const Viseme& rhs) const {
+	return options == rhs.options;
+}
+
+bool Viseme::operator!=(const Viseme& rhs) const {
+	return !operator==(rhs);
+}

src/Viseme.h

@@ -0,0 +1,31 @@
+#pragma once
+#include "Shape.h"
+#include "centiseconds.h"
+#include <map>
+#include <array>
+
+class VisemeOption {
+public:
+	VisemeOption(Shape shape);
+	VisemeOption(Shape nearB, Shape nearC, Shape nearD, Shape nearE, Shape nearF);
+	VisemeOption(Shape nearA, Shape nearB, Shape nearC, Shape nearD, Shape nearE, Shape nearF, Shape nearG, Shape nearH, Shape nearX);
+	Shape getShape(Shape context) const;
+	bool operator==(const VisemeOption& rhs) const;
+	bool operator!=(const VisemeOption& rhs) const;
+
+private:
+	std::array<Shape, 9> shapes;
+};
+
+class Viseme {
+public:
+	Viseme(const VisemeOption& option);
+	Viseme(const VisemeOption& option1, centiseconds threshold, const VisemeOption& option2);
+	Viseme(const VisemeOption& option1, centiseconds threshold1, const VisemeOption& option2, centiseconds threshold2, const VisemeOption& option3);
+	Shape getShape(centiseconds duration, Shape context) const;
+	bool operator==(const Viseme& rhs) const;
+	bool operator!=(const Viseme& rhs) const;
+
+private:
+	std::map<centiseconds, VisemeOption> options;
+};

src/mouthAnimation.cpp

@@ -2,57 +2,18 @@
 #include "logging.h"
 #include <unordered_set>
 #include <unordered_map>
-#include <array>
+#include <boost/algorithm/clamp.hpp>
+#include "Viseme.h"
 
 using std::map;
 using std::unordered_set;
 using std::unordered_map;
 using std::vector;
 using boost::optional;
+using std::chrono::duration_cast;
+using boost::algorithm::clamp;
 
-using AnimationResult = Timeline<Shape>;
+Timeline<Viseme> animate(optional<Phone> phone, centiseconds duration, centiseconds previousPhoneDuration) {
-
-AnimationResult animateFixedSound(Shape shape, centiseconds duration) {
-	return AnimationResult{ {0cs, duration, shape} };
-}
-
-// Diphtong vowels
-AnimationResult animateDiphtong(Shape first, Shape second, centiseconds duration) {
-	return AnimationResult{
-		{ 0cs, duration, first },
-		{ duration / 2, duration, second }
-	};
-}
-
-// P, B
-AnimationResult animateBilabialStop(centiseconds duration, centiseconds leftPhoneDuration, optional<Shape> rightShape) {
-	Shape openShape = rightShape.value_or(Shape::B);
-	if (openShape == Shape::A) {
-		openShape = Shape::B;
-	}
-
-	centiseconds closedShapeDuration = leftPhoneDuration / 2;
-	if (closedShapeDuration.count() < 4) closedShapeDuration = 4cs;
-	if (closedShapeDuration.count() > 16) closedShapeDuration = 16cs;
-
-	return AnimationResult{
-		{ -closedShapeDuration, 0cs, Shape::A },
-		{ 0cs, duration, openShape }
-	};
-}
-
-// Sounds with no fixed mouth position.
-// Mapping specifies the shape to use for every right shape.
-AnimationResult animateFlexibleSound(std::array<Shape, 7> mapping, centiseconds duration, optional<Shape> rightShape) {
-	constexpr int mapSize = std::tuple_size<decltype(mapping)>::value;
-	static_assert(static_cast<int>(Shape::EndSentinel) == mapSize, "Shape definition has changed.");
-
-	Shape right = rightShape.value_or(Shape::A);
-	Shape shape = mapping[static_cast<int>(right)];
-	return AnimationResult{ { 0cs, duration, shape } };
-}
-
-AnimationResult animate(optional<Phone> phone, centiseconds duration, centiseconds leftPhoneDuration, optional<Shape> rightShape) {
 	constexpr Shape A = Shape::A;
 	constexpr Shape B = Shape::B;
 	constexpr Shape C = Shape::C;
@@ -60,94 +21,117 @@ AnimationResult animate(optional<Phone> phone, centiseconds duration, centisecon
 	constexpr Shape E = Shape::E;
 	constexpr Shape F = Shape::F;
 	constexpr Shape G = Shape::G;
+	constexpr Shape H = Shape::H;
+	constexpr Shape X = Shape::X;
 
-	if (!phone) {
+	auto single = [&](Viseme viseme) {
-		return animateFixedSound(A, duration);
+		return Timeline<Viseme>{
-	}
+			{ 0cs, duration, viseme }
+		};
+	};
+
+	auto diphtong = [&](Viseme first, Viseme second) {
+		centiseconds firstDuration = duration_cast<centiseconds>(duration * 0.6);
+		return Timeline<Viseme>{
+			{ 0cs, firstDuration, first },
+			{ firstDuration, duration, second }
+		};
+	};
+
+	auto bilabialStop = [&]() {
+		centiseconds closedDuration = clamp(previousPhoneDuration / 2, 4cs, 16cs);
+		return Timeline<Viseme>{
+			{ -closedDuration, 0cs, { A } },
+			{ 0cs, duration, {{ B, C, D, E, F }} }
+		};
+	};
+
+	if (!phone)				return single({ X });
 
 	switch (*phone) {
-	case Phone::Unknown:	return animateFixedSound(B, duration);
+	case Phone::Unknown:	return single({ B });
-	case Phone::AO:			return animateFixedSound(E, duration);
+	case Phone::AO:			return single({ E });
-	case Phone::AA:			return animateFixedSound(D, duration);
+	case Phone::AA:			return single({ D });
-	case Phone::IY:			return animateFixedSound(B, duration);
+	case Phone::IY:			return single({ B });
-	case Phone::UW:			return animateFixedSound(F, duration);
+	case Phone::UW:			return single({ F });
-	case Phone::EH:			return animateFixedSound(C, duration);
+	case Phone::EH:			return single({ C });
-	case Phone::IH:			return animateFixedSound(B, duration);
+	case Phone::IH:			return single({ B });
-	case Phone::UH:			return animateFixedSound(E, duration);
+	case Phone::UH:			return single({ E });
-	case Phone::AH:			return animateFixedSound(C, duration);
+	case Phone::AH:			return single({ C });
-	case Phone::AE:			return animateFixedSound(D, duration);
+	case Phone::AE:			return single({ D });
-	case Phone::EY:			return animateDiphtong(C, B, duration);
+	case Phone::EY:			return diphtong({ C }, { B });
-	case Phone::AY:			return animateDiphtong(D, B, duration);
+	case Phone::AY:			return diphtong({ D }, { B });
-	case Phone::OW:			return animateDiphtong(E, F, duration);
+	case Phone::OW:			return diphtong({ E }, { F });
-	case Phone::AW:			return animateDiphtong(D, F, duration);
+	case Phone::AW:			return diphtong({ D }, { F });
-	case Phone::OY:			return animateDiphtong(E, B, duration);
+	case Phone::OY:			return diphtong({ E }, { B });
-	case Phone::ER:			return animateFixedSound(E, duration);
+	case Phone::ER:			return single({ E });
 	case Phone::P:
-	case Phone::B:			return animateBilabialStop(duration, leftPhoneDuration, rightShape);
+	case Phone::B:			return bilabialStop();
 	case Phone::T:
 	case Phone::D:
-	case Phone::K:			return animateFlexibleSound({ B, B, B, B, B, F, B }, duration, rightShape);
+	case Phone::K:			return single({ { B, B, B, B, F } });
-	case Phone::G:			return animateFlexibleSound({ B, B, C, C, E, F, B }, duration, rightShape);
+	case Phone::G:			return single({ { B, C, C, E, F } });
 	case Phone::CH:
-	case Phone::JH:			return animateFlexibleSound({ B, B, B, B, B, F, B }, duration, rightShape);
+	case Phone::JH:			return single({ { B, B, B, B, F } });
 	case Phone::F:
-	case Phone::V:			return animateFixedSound(G, duration);
+	case Phone::V:			return single({ G });
 	case Phone::TH:
 	case Phone::DH:
 	case Phone::S:
 	case Phone::Z:
 	case Phone::SH:
-	case Phone::ZH:			return animateFlexibleSound({ B, B, B, B, B, F, B }, duration, rightShape);
+	case Phone::ZH:			return single({ { B, B, B, B, F } });
-	case Phone::HH:			return animateFlexibleSound({ B, B, C, D, E, F, B }, duration, rightShape);
+	case Phone::HH:			return single({ { B, C, D, E, F } });
-	case Phone::M:			return animateFixedSound(A, duration);
+	case Phone::M:			return single({ A });
-	case Phone::N:			return animateFlexibleSound({ B, B, C, C, C, F, B }, duration, rightShape);
+	case Phone::N:			return single({ { B, C, C, C, F } });
-	case Phone::NG:			return animateFlexibleSound({ B, B, C, D, E, F, B }, duration, rightShape);
+	case Phone::NG:			return single({ { B, C, D, E, F } });
-	case Phone::L:			return animateFlexibleSound({ C, C, C, D, E, F, C }, duration, rightShape);
+	case Phone::L:			return single({ { H, H, H, E, F } });
-	case Phone::R:			return animateFlexibleSound({ B, B, B, B, B, F, B }, duration, rightShape);
+	case Phone::R:			return single({ { B, B, B, B, F } });
-	case Phone::Y:			return animateFixedSound(B, duration);
+	case Phone::Y:			return single({ B });
-	case Phone::W:			return animateFixedSound(F, duration);
+	case Phone::W:			return single({ F });
 	default:
 		throw std::invalid_argument("Unexpected phone.");
 	}
 }
 
 ContinuousTimeline<Shape> animate(const BoundedTimeline<Phone> &phones) {
-	// Convert phones to continuous timeline so that silences show up when iterating
+	// Convert phones to continuous timeline so that silences aren't skipped when iterating
 	ContinuousTimeline<optional<Phone>> continuousPhones(phones.getRange(), boost::none);
 	for (const auto& timedPhone : phones) {
 		continuousPhones.set(timedPhone.getTimeRange(), timedPhone.getValue());
 	}
 
-	ContinuousTimeline<Shape> shapes(phones.getRange(), Shape::A);
+	// Create timeline of visemes
-
+	ContinuousTimeline<Viseme> visemes(phones.getRange(), { Shape::X });
-	// Iterate phones in *reverse* order so we can access the right-hand result
+	centiseconds previousPhoneDuration = 0cs;
-	optional<Shape> lastShape;
+	for (const auto& timedPhone : continuousPhones) {
-	centiseconds cutoff = shapes.getRange().getEnd();
-	for (auto it = continuousPhones.rbegin(); it != continuousPhones.rend(); ++it) {
 		// Animate one phone
-		optional<Phone> phone = it->getValue();
+		optional<Phone> phone = timedPhone.getValue();
-		centiseconds duration = it->getTimeRange().getLength();
+		centiseconds duration = timedPhone.getTimeRange().getLength();
-		bool hasLeftPhone = std::next(it) != continuousPhones.rend() && std::next(it)->getEnd() == it->getStart();
+		Timeline<Viseme> phoneVisemes = animate(phone, duration, previousPhoneDuration);
-		centiseconds leftPhoneDuration = hasLeftPhone
-			? std::next(it)->getTimeRange().getLength()
-			: 0cs;
-		Timeline<Shape> result = animate(phone, duration, leftPhoneDuration, lastShape);
 
 		// Result timing is relative to phone. Make absolute.
-		result.shift(it->getStart());
+		phoneVisemes.shift(timedPhone.getStart());
 
-		// New shapes must not overwrite existing shapes
+		// Copy to viseme timeline
-		result.clear(cutoff, shapes.getRange().getEnd());
+		for (const auto& timedViseme : phoneVisemes) {
-
+			visemes.set(timedViseme);
-		// Copy to target timeline
-		for (const auto& timedShape : result) {
-			shapes.set(timedShape);
 		}
 
-		lastShape = result.empty() ? optional<Shape>() : result.begin()->getValue();
+		previousPhoneDuration = duration;
-		if (!result.empty()) {
+	}
-			cutoff = result.begin()->getStart();
+
-		}
+	// Create timeline of shapes.
+	// Iterate visemes in *reverse* order so we always know what shape will follow.
+	ContinuousTimeline<Shape> shapes(phones.getRange(), Shape::X);
+	Shape lastShape = Shape::X;
+	for (auto it = visemes.rbegin(); it != visemes.rend(); ++it) {
+		Viseme viseme = it->getValue();
+
+		// Convert viseme to phone
+		Shape shape = viseme.getShape(it->getTimeRange().getLength(), lastShape);
+
+		shapes.set(it->getTimeRange(), shape);
 	}
 
 	for (const auto& timedShape : shapes) {