From 4af606ae89964eda9ff115cc8aac23735bba92bc Mon Sep 17 00:00:00 2001 From: Daniel Wolf Date: Tue, 13 Dec 2016 19:59:05 +0100 Subject: [PATCH] Optimizing timing to make the animation less jittery and more readable --- CMakeLists.txt | 2 + extras/SonyVegas/Debug Rhubarb.cs | 3 +- src/animation/mouthAnimation.cpp | 14 ++- src/animation/timingOptimization.cpp | 159 +++++++++++++++++++++++++++ src/animation/timingOptimization.h | 8 ++ 5 files changed, 179 insertions(+), 7 deletions(-) create mode 100644 src/animation/timingOptimization.cpp create mode 100644 src/animation/timingOptimization.h diff --git a/CMakeLists.txt b/CMakeLists.txt index cbd927e..30b4318 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -229,6 +229,8 @@ add_library(rhubarb-animation src/animation/shapeRule.cpp src/animation/shapeRule.h src/animation/shapeShorthands.h + src/animation/timingOptimization.cpp + src/animation/timingOptimization.h src/animation/tweening.cpp src/animation/tweening.h ) diff --git a/extras/SonyVegas/Debug Rhubarb.cs b/extras/SonyVegas/Debug Rhubarb.cs index c13fa76..61c5f3a 100644 --- a/extras/SonyVegas/Debug Rhubarb.cs +++ b/extras/SonyVegas/Debug Rhubarb.cs @@ -260,7 +260,8 @@ public enum EventType { Word, RawPhone, Phone, - Shape + Shape, + Segment } public enum VisualizationType { diff --git a/src/animation/mouthAnimation.cpp b/src/animation/mouthAnimation.cpp index e7ad31b..f2ce096 100644 --- a/src/animation/mouthAnimation.cpp +++ b/src/animation/mouthAnimation.cpp @@ -4,19 +4,21 @@ #include "roughAnimation.h" #include "pauseAnimation.h" #include "tweening.h" +#include "timingOptimization.h" JoiningContinuousTimeline animate(const BoundedTimeline &phones) { // Create timeline of shape rules const ContinuousTimeline shapeRules = getShapeRules(phones); - // Animate - JoiningContinuousTimeline shapes = animateRough(shapeRules); - shapes = animatePauses(shapes); - shapes = insertTweens(shapes); + // Animate in multiple steps + JoiningContinuousTimeline animation = animateRough(shapeRules); + animation = optimizeTiming(animation); + animation = animatePauses(animation); + animation = insertTweens(animation); - for (const auto& timedShape : shapes) { + for (const auto& timedShape : animation) { logTimedEvent("shape", timedShape); } - return shapes; + return animation; } diff --git a/src/animation/timingOptimization.cpp b/src/animation/timingOptimization.cpp new file mode 100644 index 0000000..b390871 --- /dev/null +++ b/src/animation/timingOptimization.cpp @@ -0,0 +1,159 @@ +#include "timingOptimization.h" +#include "timedLogging.h" +#include +#include +#include +#include "shapeRule.h" + +using std::string; +using std::map; + +string getShapesString(const JoiningContinuousTimeline& shapes) { + string result; + for (const auto& timedShape : shapes) { + if (result.size()) { + result.append(" "); + } + result.append(boost::lexical_cast(timedShape.getValue())); + } + return result; +} + +// Modifies the timing of the given animation to fit into the specified target time range without jitter. +JoiningContinuousTimeline retime(const JoiningContinuousTimeline& sourceShapes, TimeRange targetRange) { + logTimedEvent("segment", targetRange, getShapesString(sourceShapes)); + + JoiningContinuousTimeline targetShapes(targetRange, Shape::X); + if (sourceShapes.empty()) return targetShapes; + + // Too short, and and we get flickering. Too long, and too many shapes are lost. + // Good values turn out to be 5 to 7 cs, with 7 cs sometimes looking just marginally better. + const centiseconds minShapeDuration = 7_cs; + + // Animate backwards + // ... `targetPosition` points to the earliest target shape already written + centiseconds targetPosition = targetRange.getEnd(); + while (targetPosition > targetRange.getStart()) { + // Determine the time range of source shapes competing for the next target shape + TimeRange candidateRange(targetPosition - minShapeDuration, targetPosition); + if (targetPosition == targetRange.getEnd()) { + // This is the first iteration. + // Extend the candidate range to the right in order to consider all source shapes after the target range. + candidateRange.setEndIfLater(sourceShapes.getRange().getEnd()); + } + if (candidateRange.getStart() >= sourceShapes.getRange().getEnd()) { + // We haven't reached the source range yet. + // Extend the candidate range to the left in order to encompass the first (last) source shape. + candidateRange.setStart(sourceShapes.rbegin()->getStart()); + } + + // Collect candidate shapes with weights + const BoundedTimeline candidateShapes(candidateRange, sourceShapes); + map candidateShapeWeights; + for (const auto& timedShape : candidateShapes) { + candidateShapeWeights[timedShape.getValue()] += timedShape.getDuration(); + } + + // Select shape with highest total duration within the candidate range + const Shape targetShape = std::max_element( + candidateShapeWeights.begin(), candidateShapeWeights.end(), + [](auto a, auto b) { return a.second < b.second; } + )->first; + + // Determine how long to display the shape + TimeRange targetShapeRange(candidateRange.getStart(), targetPosition); + if (targetShape == candidateShapes.begin()->getValue()) { + // We've chosen the very first candidate shape. Let's start at the beginning of the shape. + targetShapeRange.setStartIfEarlier(candidateShapes.begin()->getStart()); + } + if (targetShapeRange.getStart() <= sourceShapes.getRange().getStart()) { + // We've used up the last (first) source shape. Fill the entire remaining target range. + targetShapeRange.setStart(targetRange.getStart()); + } + targetShapeRange.trimLeft(targetRange.getStart()); + + // Draw shape + targetShapes.set(targetShapeRange, targetShape); + + targetPosition = targetShapeRange.getStart(); + } + + return targetShapes; +} + +JoiningContinuousTimeline retime(const JoiningContinuousTimeline& shapes, TimeRange sourceRange, TimeRange targetRange) { + const auto sourceShapes = JoiningContinuousTimeline(sourceRange, Shape::X, shapes); + return retime(sourceShapes, targetRange); +} + +enum class MouthState { + Idle, + Closed, + Open +}; + +JoiningContinuousTimeline optimizeTiming(const JoiningContinuousTimeline& shapes) { + // Identify segments with idle, closed, and open mouth shapes + JoiningContinuousTimeline segments(shapes.getRange(), MouthState::Idle); + for (const auto& timedShape : shapes) { + const Shape shape = timedShape.getValue(); + const MouthState mouthState = shape == Shape::X ? MouthState::Idle : shape == Shape::A ? MouthState::Closed : MouthState::Open; + segments.set(timedShape.getTimeRange(), mouthState); + } + + // The minimum duration a segment of open or closed mouth shapes must have to visually register + const centiseconds minSegmentDuration = 8_cs; + // The maximum amount by which the start of a shape can be brought forward + const centiseconds maxExtensionDuration = 6_cs; + + // Make sure all open and closed segments are long enough to register visually. + // We don't care about idle shapes at this point. + JoiningContinuousTimeline result(shapes.getRange(), Shape::X); + // ... we're filling the result timeline from right to left, so `resultStart` points to the earliest shape already written + centiseconds resultStart = result.getRange().getEnd(); + for (auto segmentIt = segments.rbegin(); segmentIt != segments.rend(); ++segmentIt) { + if (segmentIt->getValue() == MouthState::Idle) continue; + + const centiseconds segmentTargetEnd = std::min(segmentIt->getEnd(), resultStart); + if (segmentTargetEnd - segmentIt->getStart() >= minSegmentDuration) { + // The segment is long enough; we don't have to extend it to the left. + const TimeRange targetRange(segmentIt->getStart(), segmentTargetEnd); + const auto retimedSegment = retime(shapes, segmentIt->getTimeRange(), targetRange); + for (const auto& timedShape : retimedSegment) { + result.set(timedShape); + } + resultStart = targetRange.getStart(); + } else { + // The segment is too short; we have to extend it to the left. + // Find all adjacent segments to our left that are also too short, then distribute them evenly. + const auto begin = segmentIt; + auto end = std::next(begin); + while (end != segments.rend() && end->getValue() != MouthState::Idle && end->getDuration() < minSegmentDuration) ++end; + + // Determine how much we should extend the entire set of short segments to the left + const size_t shortSegmentCount = std::distance(begin, end); + const centiseconds desiredDuration = minSegmentDuration * shortSegmentCount; + const centiseconds currentDuration = begin->getEnd() - std::prev(end)->getStart(); + const centiseconds desiredExtensionDuration = desiredDuration - currentDuration; + const centiseconds availableExtensionDuration = end != segments.rend() ? end->getDuration() - 1_cs : 0_cs; + const centiseconds extensionDuration = std::min({desiredExtensionDuration, availableExtensionDuration, maxExtensionDuration}); + + // Distribute available time range evenly among all short segments + const centiseconds shortSegmentsTargetStart = std::prev(end)->getStart() - extensionDuration; + for (auto shortSegmentIt = begin; shortSegmentIt != end; ++shortSegmentIt) { + size_t remainingShortSegmentCount = std::distance(shortSegmentIt, end); + const centiseconds segmentDuration = (resultStart - shortSegmentsTargetStart) / remainingShortSegmentCount; + const TimeRange segmentTargetRange(resultStart - segmentDuration, resultStart); + const auto retimedSegment = retime(shapes, shortSegmentIt->getTimeRange(), segmentTargetRange); + for (const auto& timedShape : retimedSegment) { + result.set(timedShape); + } + resultStart = segmentTargetRange.getStart(); + } + + segmentIt = std::prev(end); + } + } + + return result; +} diff --git a/src/animation/timingOptimization.h b/src/animation/timingOptimization.h new file mode 100644 index 0000000..b02219f --- /dev/null +++ b/src/animation/timingOptimization.h @@ -0,0 +1,8 @@ +#pragma once + +#include "Shape.h" +#include "ContinuousTimeline.h" + +// Changes the timing of an existing animation to reduce jitter and to make sure all shapes register visually. +// In some cases, shapes may be omitted. +JoiningContinuousTimeline optimizeTiming(const JoiningContinuousTimeline& shapes);