Optimizing timing to make the animation less jittery and more readable

CMakeLists.txt

@@ -229,6 +229,8 @@ add_library(rhubarb-animation
 	src/animation/shapeRule.cpp
 	src/animation/shapeRule.h
 	src/animation/shapeShorthands.h
+	src/animation/timingOptimization.cpp
+	src/animation/timingOptimization.h
 	src/animation/tweening.cpp
 	src/animation/tweening.h
 )

extras/SonyVegas/Debug Rhubarb.cs

@@ -260,7 +260,8 @@ public enum EventType {
 	Word,
 	RawPhone,
 	Phone,
-	Shape
+	Shape,
+	Segment
 }
 
 public enum VisualizationType {

src/animation/mouthAnimation.cpp

@@ -4,19 +4,21 @@
 #include "roughAnimation.h"
 #include "pauseAnimation.h"
 #include "tweening.h"
+#include "timingOptimization.h"
 
 JoiningContinuousTimeline<Shape> animate(const BoundedTimeline<Phone> &phones) {
 	// Create timeline of shape rules
 	const ContinuousTimeline<ShapeRule> shapeRules = getShapeRules(phones);
 
-	// Animate
+	// Animate in multiple steps
-	JoiningContinuousTimeline<Shape> shapes = animateRough(shapeRules);
+	JoiningContinuousTimeline<Shape> animation = animateRough(shapeRules);
-	shapes = animatePauses(shapes);
+	animation = optimizeTiming(animation);
-	shapes = insertTweens(shapes);
+	animation = animatePauses(animation);
+	animation = insertTweens(animation);
 
-	for (const auto& timedShape : shapes) {
+	for (const auto& timedShape : animation) {
 		logTimedEvent("shape", timedShape);
 	}
 
-	return shapes;
+	return animation;
 }

src/animation/timingOptimization.cpp

@@ -0,0 +1,159 @@
+#include "timingOptimization.h"
+#include "timedLogging.h"
+#include <boost/lexical_cast.hpp>
+#include <map>
+#include <algorithm>
+#include "shapeRule.h"
+
+using std::string;
+using std::map;
+
+string getShapesString(const JoiningContinuousTimeline<Shape>& shapes) {
+	string result;
+	for (const auto& timedShape : shapes) {
+		if (result.size()) {
+			result.append(" ");
+		}
+		result.append(boost::lexical_cast<std::string>(timedShape.getValue()));
+	}
+	return result;
+}
+
+// Modifies the timing of the given animation to fit into the specified target time range without jitter.
+JoiningContinuousTimeline<Shape> retime(const JoiningContinuousTimeline<Shape>& sourceShapes, TimeRange targetRange) {
+	logTimedEvent("segment", targetRange, getShapesString(sourceShapes));
+
+	JoiningContinuousTimeline<Shape> targetShapes(targetRange, Shape::X);
+	if (sourceShapes.empty()) return targetShapes;
+
+	// Too short, and and we get flickering. Too long, and too many shapes are lost.
+	// Good values turn out to be 5 to 7 cs, with 7 cs sometimes looking just marginally better.
+	const centiseconds minShapeDuration = 7_cs;
+
+	// Animate backwards
+	// ... `targetPosition` points to the earliest target shape already written
+	centiseconds targetPosition = targetRange.getEnd();
+	while (targetPosition > targetRange.getStart()) {
+		// Determine the time range of source shapes competing for the next target shape
+		TimeRange candidateRange(targetPosition - minShapeDuration, targetPosition);
+		if (targetPosition == targetRange.getEnd()) {
+			// This is the first iteration.
+			// Extend the candidate range to the right in order to consider all source shapes after the target range.
+			candidateRange.setEndIfLater(sourceShapes.getRange().getEnd());
+		}
+		if (candidateRange.getStart() >= sourceShapes.getRange().getEnd()) {
+			// We haven't reached the source range yet.
+			// Extend the candidate range to the left in order to encompass the first (last) source shape.
+			candidateRange.setStart(sourceShapes.rbegin()->getStart());
+		}
+
+		// Collect candidate shapes with weights
+		const BoundedTimeline<Shape> candidateShapes(candidateRange, sourceShapes);
+		map<Shape, centiseconds> candidateShapeWeights;
+		for (const auto& timedShape : candidateShapes) {
+			candidateShapeWeights[timedShape.getValue()] += timedShape.getDuration();
+		}
+
+		// Select shape with highest total duration within the candidate range
+		const Shape targetShape = std::max_element(
+			candidateShapeWeights.begin(), candidateShapeWeights.end(),
+			[](auto a, auto b) { return a.second < b.second; }
+		)->first;
+
+		// Determine how long to display the shape
+		TimeRange targetShapeRange(candidateRange.getStart(), targetPosition);
+		if (targetShape == candidateShapes.begin()->getValue()) {
+			// We've chosen the very first candidate shape. Let's start at the beginning of the shape.
+			targetShapeRange.setStartIfEarlier(candidateShapes.begin()->getStart());
+		}
+		if (targetShapeRange.getStart() <= sourceShapes.getRange().getStart()) {
+			// We've used up the last (first) source shape. Fill the entire remaining target range.
+			targetShapeRange.setStart(targetRange.getStart());
+		}
+		targetShapeRange.trimLeft(targetRange.getStart());
+
+		// Draw shape
+		targetShapes.set(targetShapeRange, targetShape);
+
+		targetPosition = targetShapeRange.getStart();
+	}
+
+	return targetShapes;
+}
+
+JoiningContinuousTimeline<Shape> retime(const JoiningContinuousTimeline<Shape>& shapes, TimeRange sourceRange, TimeRange targetRange) {
+	const auto sourceShapes = JoiningContinuousTimeline<Shape>(sourceRange, Shape::X, shapes);
+	return retime(sourceShapes, targetRange);
+}
+
+enum class MouthState {
+	Idle,
+	Closed,
+	Open
+};
+
+JoiningContinuousTimeline<Shape> optimizeTiming(const JoiningContinuousTimeline<Shape>& shapes) {
+	// Identify segments with idle, closed, and open mouth shapes
+	JoiningContinuousTimeline<MouthState> segments(shapes.getRange(), MouthState::Idle);
+	for (const auto& timedShape : shapes) {
+		const Shape shape = timedShape.getValue();
+		const MouthState mouthState = shape == Shape::X ? MouthState::Idle : shape == Shape::A ? MouthState::Closed : MouthState::Open;
+		segments.set(timedShape.getTimeRange(), mouthState);
+	}
+
+	// The minimum duration a segment of open or closed mouth shapes must have to visually register
+	const centiseconds minSegmentDuration = 8_cs;
+	// The maximum amount by which the start of a shape can be brought forward
+	const centiseconds maxExtensionDuration = 6_cs;
+
+	// Make sure all open and closed segments are long enough to register visually.
+	// We don't care about idle shapes at this point.
+	JoiningContinuousTimeline<Shape> result(shapes.getRange(), Shape::X);
+	// ... we're filling the result timeline from right to left, so `resultStart` points to the earliest shape already written
+	centiseconds resultStart = result.getRange().getEnd();
+	for (auto segmentIt = segments.rbegin(); segmentIt != segments.rend(); ++segmentIt) {
+		if (segmentIt->getValue() == MouthState::Idle) continue;
+
+		const centiseconds segmentTargetEnd = std::min(segmentIt->getEnd(), resultStart);
+		if (segmentTargetEnd - segmentIt->getStart() >= minSegmentDuration) {
+			// The segment is long enough; we don't have to extend it to the left.
+			const TimeRange targetRange(segmentIt->getStart(), segmentTargetEnd);
+			const auto retimedSegment = retime(shapes, segmentIt->getTimeRange(), targetRange);
+			for (const auto& timedShape : retimedSegment) {
+				result.set(timedShape);
+			}
+			resultStart = targetRange.getStart();
+		} else {
+			// The segment is too short; we have to extend it to the left.
+			// Find all adjacent segments to our left that are also too short, then distribute them evenly.
+			const auto begin = segmentIt;
+			auto end = std::next(begin);
+			while (end != segments.rend() && end->getValue() != MouthState::Idle && end->getDuration() < minSegmentDuration) ++end;
+
+			// Determine how much we should extend the entire set of short segments to the left
+			const size_t shortSegmentCount = std::distance(begin, end);
+			const centiseconds desiredDuration = minSegmentDuration * shortSegmentCount;
+			const centiseconds currentDuration = begin->getEnd() - std::prev(end)->getStart();
+			const centiseconds desiredExtensionDuration = desiredDuration - currentDuration;
+			const centiseconds availableExtensionDuration = end != segments.rend() ? end->getDuration() - 1_cs : 0_cs;
+			const centiseconds extensionDuration = std::min({desiredExtensionDuration, availableExtensionDuration, maxExtensionDuration});
+
+			// Distribute available time range evenly among all short segments
+			const centiseconds shortSegmentsTargetStart = std::prev(end)->getStart() - extensionDuration;
+			for (auto shortSegmentIt = begin; shortSegmentIt != end; ++shortSegmentIt) {
+				size_t remainingShortSegmentCount = std::distance(shortSegmentIt, end);
+				const centiseconds segmentDuration = (resultStart - shortSegmentsTargetStart) / remainingShortSegmentCount;
+				const TimeRange segmentTargetRange(resultStart - segmentDuration, resultStart);
+				const auto retimedSegment = retime(shapes, shortSegmentIt->getTimeRange(), segmentTargetRange);
+				for (const auto& timedShape : retimedSegment) {
+					result.set(timedShape);
+				}
+				resultStart = segmentTargetRange.getStart();
+			}
+
+			segmentIt = std::prev(end);
+		}
+	}
+
+	return result;
+}

src/animation/timingOptimization.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#include "Shape.h"
+#include "ContinuousTimeline.h"
+
+// Changes the timing of an existing animation to reduce jitter and to make sure all shapes register visually.
+// In some cases, shapes may be omitted.
+JoiningContinuousTimeline<Shape> optimizeTiming(const JoiningContinuousTimeline<Shape>& shapes);