jamosdev
/
rhubarb-lip-sync
mirror of https://github.com/DanielSWolf/rhubarb-lip-sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extracted animation functions into dedicated files

This commit is contained in:
Daniel Wolf 2016-12-11 21:38:14 +01:00
parent 651521a8c1
commit 43cf341586
11 changed files with 227 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CMakeLists.txt
View File

@ -222,7 +222,15 @@ add_library(rhubarb-animation
src/animation/animationRules.h
src/animation/mouthAnimation.cpp
src/animation/mouthAnimation.h
src/animation/pauseAnimation.cpp
src/animation/pauseAnimation.h
src/animation/roughAnimation.cpp
src/animation/roughAnimation.h
src/animation/shapeRule.cpp
src/animation/shapeRule.h
src/animation/shapeShorthands.h
src/animation/tweening.cpp
src/animation/tweening.h
)
target_include_directories(rhubarb-animation PUBLIC "src/animation")
target_link_libraries(rhubarb-animation

1
src/animation/animationRules.cpp
View File

@ -2,6 +2,7 @@
#include <boost/algorithm/clamp.hpp>
#include "shapeShorthands.h"
#include "array.h"
#include "ContinuousTimeline.h"
using std::chrono::duration_cast;
using boost::algorithm::clamp;

193
src/animation/mouthAnimation.cpp
View File

@ -1,194 +1,9 @@
#include "mouthAnimation.h"
#include "logging.h"
#include <unordered_set>
#include <unordered_map>
#include <boost/algorithm/clamp.hpp>
#include <boost/range/adaptor/transformed.hpp>
#include "timedLogging.h"
#include "shapeShorthands.h"
#include "animationRules.h"
using std::map;
using std::unordered_set;
using std::unordered_map;
using std::vector;
using boost::optional;
using boost::make_optional;
using std::chrono::duration_cast;
using boost::algorithm::clamp;
using boost::adaptors::transformed;
using std::pair;
using std::tuple;
JoiningContinuousTimeline<Shape> insertTweens(JoiningContinuousTimeline<Shape> shapes) {
centiseconds minTweenDuration = 4_cs;
centiseconds maxTweenDuration = 10_cs;
JoiningContinuousTimeline<Shape> result(shapes);
for (auto first = shapes.begin(), second = std::next(shapes.begin());
first != shapes.end() && second != shapes.end();
++first, ++second)
{
auto pair = getTween(first->getValue(), second->getValue());
if (!pair) continue;
Shape tweenShape;
TweenTiming tweenTiming;
std::tie(tweenShape, tweenTiming) = *pair;
TimeRange firstTimeRange = first->getTimeRange();
TimeRange secondTimeRange = second->getTimeRange();
centiseconds tweenStart, tweenDuration;
switch (tweenTiming) {
case TweenTiming::Early: {
tweenDuration = std::min(firstTimeRange.getDuration() / 3, maxTweenDuration);
tweenStart = firstTimeRange.getEnd() - tweenDuration;
break;
}
case TweenTiming::Centered: {
tweenDuration = std::min({ firstTimeRange.getDuration() / 3, secondTimeRange.getDuration() / 3, maxTweenDuration });
tweenStart = firstTimeRange.getEnd() - tweenDuration / 2;
break;
}
case TweenTiming::Late: {
tweenDuration = std::min(secondTimeRange.getDuration() / 3, maxTweenDuration);
tweenStart = secondTimeRange.getStart();
break;
}
}
if (tweenDuration < minTweenDuration) continue;
result.set(tweenStart, tweenStart + tweenDuration, tweenShape);
}
return result;
}
JoiningContinuousTimeline<Shape> animatePauses(const JoiningContinuousTimeline<Shape>& shapes) {
JoiningContinuousTimeline<Shape> result(shapes);
// Don't close mouth for short pauses
for_each_adjacent(shapes.begin(), shapes.end(), [&](const Timed<Shape>& lhs, const Timed<Shape>& pause, const Timed<Shape>& rhs) {
if (pause.getValue() != X) return;
const centiseconds maxPausedOpenMouthDuration = 35_cs;
const TimeRange timeRange = pause.getTimeRange();
if (timeRange.getDuration() <= maxPausedOpenMouthDuration) {
result.set(timeRange, getRelaxedBridge(lhs.getValue(), rhs.getValue()));
}
});
// Keep mouth open into pause if it just opened
for_each_adjacent(shapes.begin(), shapes.end(), [&](const Timed<Shape>& secondLast, const Timed<Shape>& last, const Timed<Shape>& pause) {
if (pause.getValue() != X) return;
centiseconds lastDuration = last.getDuration();
const centiseconds minOpenDuration = 20_cs;
if (isClosed(secondLast.getValue()) && !isClosed(last.getValue()) && lastDuration < minOpenDuration) {
const centiseconds minSpillDuration = 20_cs;
centiseconds spillDuration = std::min(minSpillDuration, pause.getDuration());
result.set(pause.getStart(), pause.getStart() + spillDuration, getRelaxedBridge(last.getValue(), X));
}
});
return result;
}
template<typename T, bool AutoJoin>
ContinuousTimeline<optional<T>, AutoJoin> boundedTimelinetoContinuousOptional(const BoundedTimeline<T, AutoJoin>& timeline) {
return {
timeline.getRange(), boost::none,
timeline | transformed([](const Timed<T>& timedValue) { return Timed<optional<T>>(timedValue.getTimeRange(), timedValue.getValue()); })
};
}
using ShapeRule = tuple<ShapeSet, optional<Phone>>;
ContinuousTimeline<ShapeRule> getShapeRules(const BoundedTimeline<Phone>& phones) {
// Convert to continuous timeline so that silences aren't skipped when iterating
auto continuousPhones = boundedTimelinetoContinuousOptional(phones);
// Create timeline of shape rules
ContinuousTimeline<ShapeRule> shapeRules(phones.getRange(), {{X}, boost::none});
centiseconds previousDuration = 0_cs;
for (const auto& timedPhone : continuousPhones) {
optional<Phone> phone = timedPhone.getValue();
centiseconds duration = timedPhone.getDuration();
if (phone) {
// Animate one phone
Timeline<ShapeSet> phoneShapeSets = getShapeSets(*phone, duration, previousDuration);
// Result timing is relative to phone. Make absolute.
phoneShapeSets.shift(timedPhone.getStart());
// Copy to timeline.
// Later shape sets may overwrite earlier ones if overlapping.
for (const auto& timedShapeSet : phoneShapeSets) {
shapeRules.set(timedShapeSet.getTimeRange(), {timedShapeSet.getValue(), phone});
}
}
previousDuration = duration;
}
return shapeRules;
}
// Create timeline of shapes using a bidirectional algorithm.
// Here's a rough sketch:
//
// * Most consonants result in shape sets with multiple options; most vowels have only one shape option.
// * When speaking, we tend to slur mouth shapes into each other. So we animate from start to end,
// always choosing a shape from the current set that resembles the last shape and is somewhat relaxed.
// * When speaking, we anticipate vowels, trying to form their shape before the actual vowel.
// So whenever we come across a one-shape vowel, we backtrack a little, spreating that shape to the left.
JoiningContinuousTimeline<Shape> animateRough(const ContinuousTimeline<ShapeRule>& shapeRules) {
JoiningContinuousTimeline<Shape> shapes(shapeRules.getRange(), X);
Shape referenceShape = X;
// Animate forwards
centiseconds lastAnticipatedShapeStart = -1_cs;
for (auto it = shapeRules.begin(); it != shapeRules.end(); ++it) {
const ShapeRule shapeRule = it->getValue();
const ShapeSet shapeSet = std::get<ShapeSet>(shapeRule);
const Shape shape = getClosestShape(referenceShape, shapeSet);
shapes.set(it->getTimeRange(), shape);
const auto phone = std::get<optional<Phone>>(shapeRule);
const bool anticipateShape = phone && isVowel(*phone) && shapeSet.size() == 1;
if (anticipateShape) {
// Animate backwards a little
const Shape anticipatedShape = shape;
const centiseconds anticipatedShapeStart = it->getStart();
referenceShape = anticipatedShape;
for (auto reverseIt = it; reverseIt != shapeRules.begin(); ) {
--reverseIt;
// Make sure we haven't animated too far back
centiseconds anticipatingShapeStart = reverseIt->getStart();
if (anticipatingShapeStart == lastAnticipatedShapeStart) break;
const centiseconds maxAnticipationDuration = 20_cs;
const centiseconds anticipationDuration = anticipatedShapeStart - anticipatingShapeStart;
if (anticipationDuration > maxAnticipationDuration) break;
// Make sure the new, backwards-animated shape still resembles the anticipated shape
const Shape anticipatingShape = getClosestShape(referenceShape, std::get<ShapeSet>(reverseIt->getValue()));
if (getBasicShape(anticipatingShape) != getBasicShape(anticipatedShape)) break;
// Overwrite forward-animated shape with backwards-animated, anticipating shape
shapes.set(reverseIt->getTimeRange(), anticipatingShape);
referenceShape = anticipatingShape;
}
lastAnticipatedShapeStart = anticipatedShapeStart;
}
referenceShape = anticipateShape ? shape : relax(shape);
}
return shapes;
}
#include "shapeRule.h"
#include "roughAnimation.h"
#include "pauseAnimation.h"
#include "tweening.h"
JoiningContinuousTimeline<Shape> animate(const BoundedTimeline<Phone> &phones) {
// Create timeline of shape rules

32
src/animation/pauseAnimation.cpp Normal file
View File

@ -0,0 +1,32 @@
#include "pauseAnimation.h"
#include "animationRules.h"
JoiningContinuousTimeline<Shape> animatePauses(const JoiningContinuousTimeline<Shape>& shapes) {
JoiningContinuousTimeline<Shape> result(shapes);
// Don't close mouth for short pauses
for_each_adjacent(shapes.begin(), shapes.end(), [&](const Timed<Shape>& lhs, const Timed<Shape>& pause, const Timed<Shape>& rhs) {
if (pause.getValue() != Shape::X) return;
const centiseconds maxPausedOpenMouthDuration = 35_cs;
const TimeRange timeRange = pause.getTimeRange();
if (timeRange.getDuration() <= maxPausedOpenMouthDuration) {
result.set(timeRange, getRelaxedBridge(lhs.getValue(), rhs.getValue()));
}
});
// Keep mouth open into pause if it just opened
for_each_adjacent(shapes.begin(), shapes.end(), [&](const Timed<Shape>& secondLast, const Timed<Shape>& last, const Timed<Shape>& pause) {
if (pause.getValue() != Shape::X) return;
centiseconds lastDuration = last.getDuration();
const centiseconds minOpenDuration = 20_cs;
if (isClosed(secondLast.getValue()) && !isClosed(last.getValue()) && lastDuration < minOpenDuration) {
const centiseconds minSpillDuration = 20_cs;
centiseconds spillDuration = std::min(minSpillDuration, pause.getDuration());
result.set(pause.getStart(), pause.getStart() + spillDuration, getRelaxedBridge(last.getValue(), Shape::X));
}
});
return result;
}

7
src/animation/pauseAnimation.h Normal file
View File

@ -0,0 +1,7 @@
#pragma once
#include "Shape.h"
#include "ContinuousTimeline.h"
// Takes an existing animation and modifies the pauses (X shapes) to look better.
JoiningContinuousTimeline<Shape> animatePauses(const JoiningContinuousTimeline<Shape>& shapes);

57
src/animation/roughAnimation.cpp Normal file
View File

@ -0,0 +1,57 @@
#include "roughAnimation.h"
#include <boost/optional.hpp>
using boost::optional;
// Create timeline of shapes using a bidirectional algorithm.
// Here's a rough sketch:
//
// * Most consonants result in shape sets with multiple options; most vowels have only one shape option.
// * When speaking, we tend to slur mouth shapes into each other. So we animate from start to end,
// always choosing a shape from the current set that resembles the last shape and is somewhat relaxed.
// * When speaking, we anticipate vowels, trying to form their shape before the actual vowel.
// So whenever we come across a one-shape vowel, we backtrack a little, spreating that shape to the left.
JoiningContinuousTimeline<Shape> animateRough(const ContinuousTimeline<ShapeRule>& shapeRules) {
JoiningContinuousTimeline<Shape> shapes(shapeRules.getRange(), Shape::X);
Shape referenceShape = Shape::X;
// Animate forwards
centiseconds lastAnticipatedShapeStart = -1_cs;
for (auto it = shapeRules.begin(); it != shapeRules.end(); ++it) {
const ShapeRule shapeRule = it->getValue();
const ShapeSet shapeSet = std::get<ShapeSet>(shapeRule);
const Shape shape = getClosestShape(referenceShape, shapeSet);
shapes.set(it->getTimeRange(), shape);
const auto phone = std::get<optional<Phone>>(shapeRule);
const bool anticipateShape = phone && isVowel(*phone) && shapeSet.size() == 1;
if (anticipateShape) {
// Animate backwards a little
const Shape anticipatedShape = shape;
const centiseconds anticipatedShapeStart = it->getStart();
referenceShape = anticipatedShape;
for (auto reverseIt = it; reverseIt != shapeRules.begin(); ) {
--reverseIt;
// Make sure we haven't animated too far back
centiseconds anticipatingShapeStart = reverseIt->getStart();
if (anticipatingShapeStart == lastAnticipatedShapeStart) break;
const centiseconds maxAnticipationDuration = 20_cs;
const centiseconds anticipationDuration = anticipatedShapeStart - anticipatingShapeStart;
if (anticipationDuration > maxAnticipationDuration) break;
// Make sure the new, backwards-animated shape still resembles the anticipated shape
const Shape anticipatingShape = getClosestShape(referenceShape, std::get<ShapeSet>(reverseIt->getValue()));
if (getBasicShape(anticipatingShape) != getBasicShape(anticipatedShape)) break;
// Overwrite forward-animated shape with backwards-animated, anticipating shape
shapes.set(reverseIt->getTimeRange(), anticipatingShape);
referenceShape = anticipatingShape;
}
lastAnticipatedShapeStart = anticipatedShapeStart;
}
referenceShape = anticipateShape ? shape : relax(shape);
}
return shapes;
}

6
src/animation/roughAnimation.h Normal file
View File

@ -0,0 +1,6 @@
#pragma once
#include "shapeRule.h"
// Does a rough animation (no tweening, special pause animation, etc.) using a bidirectional algorithm.
JoiningContinuousTimeline<Shape> animateRough(const ContinuousTimeline<ShapeRule>& shapeRules);

45
src/animation/shapeRule.cpp Normal file
View File

@ -0,0 +1,45 @@
#include "shapeRule.h"
#include <boost/range/adaptor/transformed.hpp>
#include "ContinuousTimeline.h"
using boost::optional;
using boost::adaptors::transformed;
template<typename T, bool AutoJoin>
ContinuousTimeline<optional<T>, AutoJoin> boundedTimelinetoContinuousOptional(const BoundedTimeline<T, AutoJoin>& timeline) {
return{
timeline.getRange(), boost::none,
timeline | transformed([](const Timed<T>& timedValue) { return Timed<optional<T>>(timedValue.getTimeRange(), timedValue.getValue()); })
};
}
ContinuousTimeline<ShapeRule> getShapeRules(const BoundedTimeline<Phone>& phones) {
// Convert to continuous timeline so that silences aren't skipped when iterating
auto continuousPhones = boundedTimelinetoContinuousOptional(phones);
// Create timeline of shape rules
ContinuousTimeline<ShapeRule> shapeRules(phones.getRange(), {{Shape::X}, boost::none});
centiseconds previousDuration = 0_cs;
for (const auto& timedPhone : continuousPhones) {
optional<Phone> phone = timedPhone.getValue();
centiseconds duration = timedPhone.getDuration();
if (phone) {
// Animate one phone
Timeline<ShapeSet> phoneShapeSets = getShapeSets(*phone, duration, previousDuration);
// Result timing is relative to phone. Make absolute.
phoneShapeSets.shift(timedPhone.getStart());
// Copy to timeline.
// Later shape sets may overwrite earlier ones if overlapping.
for (const auto& timedShapeSet : phoneShapeSets) {
shapeRules.set(timedShapeSet.getTimeRange(), {timedShapeSet.getValue(), phone});
}
}
previousDuration = duration;
}
return shapeRules;
}

12
src/animation/shapeRule.h Normal file
View File

@ -0,0 +1,12 @@
#pragma once
#include "Phone.h"
#include "animationRules.h"
#include "BoundedTimeline.h"
#include "ContinuousTimeline.h"
// A shape set with its original phone
using ShapeRule = std::tuple<ShapeSet, boost::optional<Phone>>;
// Returns shape rules for an entire timeline of phones.
ContinuousTimeline<ShapeRule> getShapeRules(const BoundedTimeline<Phone>& phones);

48
src/animation/tweening.cpp Normal file
View File

@ -0,0 +1,48 @@
#include "tweening.h"
#include "animationRules.h"
JoiningContinuousTimeline<Shape> insertTweens(const JoiningContinuousTimeline<Shape>& shapes) {
centiseconds minTweenDuration = 4_cs;
centiseconds maxTweenDuration = 10_cs;
JoiningContinuousTimeline<Shape> result(shapes);
for (auto first = shapes.begin(), second = std::next(shapes.begin());
first != shapes.end() && second != shapes.end();
++first, ++second)
{
auto pair = getTween(first->getValue(), second->getValue());
if (!pair) continue;
Shape tweenShape;
TweenTiming tweenTiming;
std::tie(tweenShape, tweenTiming) = *pair;
TimeRange firstTimeRange = first->getTimeRange();
TimeRange secondTimeRange = second->getTimeRange();
centiseconds tweenStart, tweenDuration;
switch (tweenTiming) {
case TweenTiming::Early: {
tweenDuration = std::min(firstTimeRange.getDuration() / 3, maxTweenDuration);
tweenStart = firstTimeRange.getEnd() - tweenDuration;
break;
}
case TweenTiming::Centered: {
tweenDuration = std::min({firstTimeRange.getDuration() / 3, secondTimeRange.getDuration() / 3, maxTweenDuration});
tweenStart = firstTimeRange.getEnd() - tweenDuration / 2;
break;
}
case TweenTiming::Late: {
tweenDuration = std::min(secondTimeRange.getDuration() / 3, maxTweenDuration);
tweenStart = secondTimeRange.getStart();
break;
}
}
if (tweenDuration < minTweenDuration) continue;
result.set(tweenStart, tweenStart + tweenDuration, tweenShape);
}
return result;
}

7
src/animation/tweening.h Normal file
View File

@ -0,0 +1,7 @@
#pragma once
#include "Shape.h"
#include "ContinuousTimeline.h"
// Takes an existing animation and inserts inbetween shapes for smoother results.
JoiningContinuousTimeline<Shape> insertTweens(const JoiningContinuousTimeline<Shape>& shapes);