jamosdev
/
rhubarb-lip-sync
mirror of https://github.com/DanielSWolf/rhubarb-lip-sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switched back to simple shape sets instead of shape rules 

The extra information can be generated automatically.
This commit is contained in:
Daniel Wolf 2016-12-08 11:37:48 +01:00
parent a24fe8874c
commit 21bea661c1
3 changed files with 58 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
src/animation/animationRules.cpp
View File

@ -71,45 +71,40 @@ optional<pair<Shape, TweenTiming>> getTween(Shape first, Shape second) {
return it != lookup.end() ? it->second : optional<pair<Shape, TweenTiming>>();
}
ShapeRule::ShapeRule(const ShapeSet& regularShapes, const ShapeSet& alternativeShapes) :
regularShapes(regularShapes),
alternativeShapes(alternativeShapes)
{}
Timeline<ShapeRule> getShapeRules(Phone phone, centiseconds duration, centiseconds previousDuration) {
Timeline<ShapeSet> getShapeSets(Phone phone, centiseconds duration, centiseconds previousDuration) {
// Returns a timeline with a single shape set
auto single = [duration](ShapeRule value) {
return Timeline<ShapeRule> {{0_cs, duration, value}};
auto single = [duration](ShapeSet value) {
return Timeline<ShapeSet> {{0_cs, duration, value}};
};
// Returns a timeline with two shape sets, timed as a diphthong
auto diphthong = [duration](ShapeRule first, ShapeRule second) {
auto diphthong = [duration](ShapeSet first, ShapeSet second) {
centiseconds firstDuration = duration_cast<centiseconds>(duration * 0.6);
return Timeline<ShapeRule> {
return Timeline<ShapeSet> {
{0_cs, firstDuration, first},
{firstDuration, duration, second}
};
};
// Returns a timeline with two shape sets, timed as a plosive
auto plosive = [duration, previousDuration](ShapeRule first, ShapeRule second) {
auto plosive = [duration, previousDuration](ShapeSet first, ShapeSet second) {
centiseconds minOcclusionDuration = 4_cs;
centiseconds maxOcclusionDuration = 12_cs;
centiseconds occlusionDuration = clamp(previousDuration / 2, minOcclusionDuration, maxOcclusionDuration);
return Timeline<ShapeRule> {
return Timeline<ShapeSet> {
{-occlusionDuration, 0_cs, first},
{0_cs, duration, second}
};
};
// Returns the result of `getShapeRules` when called with identical arguments
// Returns the result of `getShapeSets` when called with identical arguments
// except for a different phone.
auto like = [duration, previousDuration](Phone referencePhone) {
return getShapeRules(referencePhone, duration, previousDuration);
return getShapeSets(referencePhone, duration, previousDuration);
};
static const ShapeRule any{{A, B, C, D, E, F, G, H, X}};
static const ShapeRule anyOpen{{B, C, D, E, F, G, H}};
static const ShapeSet any{A, B, C, D, E, F, G, H, X};
static const ShapeSet anyOpen{B, C, D, E, F, G, H};
// Note:
// The shapes {A, B, G, X} are very similar. You should avoid regular shape sets containing more than one of these shapes.
@ -117,52 +112,52 @@ Timeline<ShapeRule> getShapeRules(Phone phone, centiseconds duration, centisecon
// As an exception, a very flexible rule may contain *all* these shapes.
switch (phone) {
case Phone::AO: return single({{E}});
case Phone::AA: return single({{D}});
case Phone::IY: return single({{B}, {C}});
case Phone::UW: return single({{F}});
case Phone::EH: return single({{C}});
case Phone::IH: return single({{B}, {C}});
case Phone::UH: return single({{F}});
case Phone::AH: return single({{C}});
case Phone::Schwa: return single({{B, C}});
case Phone::AE: return single({{C}});
case Phone::EY: return diphthong({{C}}, {{B}, {C}});
case Phone::AY: return duration < 20_cs ? diphthong({{C}}, {{B}, {C}}) : diphthong({{D}}, {{B}, {C}});
case Phone::OW: return single({{F}});
case Phone::AW: return duration < 30_cs ? diphthong({{C}}, {{F}}) : diphthong({{D}}, {{F}});
case Phone::OY: return diphthong({{E}}, {{B}, {C}});
case Phone::ER: return duration < 7_cs ? like(Phone::Schwa) : single({{B}});
case Phone::AO: return single({E});
case Phone::AA: return single({D});
case Phone::IY: return single({B});
case Phone::UW: return single({F});
case Phone::EH: return single({C});
case Phone::IH: return single({B});
case Phone::UH: return single({F});
case Phone::AH: return single({C});
case Phone::Schwa: return single({B, C});
case Phone::AE: return single({C});
case Phone::EY: return diphthong({C}, {B});
case Phone::AY: return duration < 20_cs ? diphthong({C}, {B}) : diphthong({D}, {B});
case Phone::OW: return single({F});
case Phone::AW: return duration < 30_cs ? diphthong({C}, {F}) : diphthong({D}, {F});
case Phone::OY: return diphthong({E}, {B});
case Phone::ER: return duration < 7_cs ? like(Phone::Schwa) : single({B});
case Phone::P:
case Phone::B: return plosive({{A}}, any);
case Phone::B: return plosive({A}, any);
case Phone::T:
case Phone::D: return plosive({{B, F}}, anyOpen);
case Phone::D: return plosive({B, F}, anyOpen);
case Phone::K:
case Phone::G: return plosive({{B, C, E, F, H}}, anyOpen);
case Phone::G: return plosive({B, C, E, F, H}, anyOpen);
case Phone::CH:
case Phone::JH: return single({{B, F}});
case Phone::JH: return single({B, F});
case Phone::F:
case Phone::V: return single({{G}});
case Phone::V: return single({G});
case Phone::TH:
case Phone::DH:
case Phone::S:
case Phone::Z:
case Phone::SH:
case Phone::ZH: return single({{B, F}});
case Phone::ZH: return single({B, F});
case Phone::HH: return single(any); // think "m-hm"
case Phone::M: return single({{A}});
case Phone::N: return single({{B, C, F, H}});
case Phone::NG: return single({{B, C, E, F}});
case Phone::L: return duration < 20_cs ? single({{B, C, E, F, H}}) : single({{H}});
case Phone::R: return single({{B, E, F}});
case Phone::Y: return single({{B, C, F}});
case Phone::W: return single({{F}});
case Phone::M: return single({A});
case Phone::N: return single({B, C, F, H});
case Phone::NG: return single({B, C, E, F});
case Phone::L: return duration < 20_cs ? single({B, C, E, F, H}) : single({H});
case Phone::R: return single({B, E, F});
case Phone::Y: return single({B, C, F});
case Phone::W: return single({F});
case Phone::Breath:
case Phone::Cough:
case Phone::Smack: return single({{C}});
case Phone::Noise: return single({{B}});
case Phone::Smack: return single({C});
case Phone::Noise: return single({B});
default: throw std::invalid_argument("Unexpected phone.");
}

29
src/animation/animationRules.h
View File

@ -14,7 +14,8 @@ Shape relax(Shape shape);
// Returns the mouth shape to use for *short* pauses between words.
Shape getRelaxedBridge(Shape lhs, Shape rhs);
// A set of mouth shapes that can be used to represent a certain sound
// A set of mouth shapes that can be used to represent a certain sound.
// The actual selection will be performed based on similarity with the previous or next shape.
using ShapeSet = std::set<Shape>;
// Gets the shape from a non-empty set of shapes that most closely resembles a reference shape.
@ -35,29 +36,7 @@ enum class TweenTiming {
// Returns the tween shape and timing to use to transition between the specified two mouth shapes.
boost::optional<std::pair<Shape, TweenTiming>> getTween(Shape first, Shape second);
// A struct describing the possible shapes to use during a given time range.
struct ShapeRule {
ShapeRule(const ShapeSet& regularShapes, const ShapeSet& alternativeShapes = {});
// A set of one or more shapes that may be used to animate a given time range.
// The actual selection will be performed based on similarity with the previous or next shape.
ShapeSet regularShapes;
// The regular animation algorithm tries to minimize mouth shape changes. As a result, the mouth may sometimes remain static for too long.
// This is a set of zero or more shapes that may be used in these cases.
// In contrast to the regular shapes, this set should only contain shapes that can be used regardless of the surrounding shapes.
ShapeSet alternativeShapes;
bool operator==(const ShapeRule& rhs) const {
return regularShapes == rhs.regularShapes && alternativeShapes == rhs.alternativeShapes;
}
bool operator!=(const ShapeRule& rhs) const {
return !operator==(rhs);
}
};
// Returns the shape rule(s) to use for a given phone.
// Returns the shape set(s) to use for a given phone.
// The resulting timeline will always cover the entire duration of the phone (starting at 0 cs).
// It may extend into the negative time range if animation is required prior to the sound being heard.
Timeline<ShapeRule> getShapeRules(Phone phone, centiseconds duration, centiseconds previousDuration);
Timeline<ShapeSet> getShapeSets(Phone phone, centiseconds duration, centiseconds previousDuration);

29
src/animation/mouthAnimation.cpp
View File

@ -104,12 +104,12 @@ ContinuousTimeline<optional<T>, AutoJoin> boundedTimelinetoContinuousOptional(co
};
}
ContinuousTimeline<ShapeRule> getShapeRules(const BoundedTimeline<Phone>& phones) {
ContinuousTimeline<ShapeSet> getShapeSets(const BoundedTimeline<Phone>& phones) {
// Convert to continuous timeline so that silences aren't skipped when iterating
auto continuousPhones = boundedTimelinetoContinuousOptional(phones);
// Create timeline of shape rules
ContinuousTimeline<ShapeRule> shapeRules(phones.getRange(), {{X}});
// Create timeline of shape sets
ContinuousTimeline<ShapeSet> shapeSets(phones.getRange(), {{X}});
centiseconds previousDuration = 0_cs;
for (const auto& timedPhone : continuousPhones) {
optional<Phone> phone = timedPhone.getValue();
@ -117,25 +117,25 @@ ContinuousTimeline<ShapeRule> getShapeRules(const BoundedTimeline<Phone>& phones
if (phone) {
// Animate one phone
Timeline<ShapeRule> phoneShapeRules = getShapeRules(*phone, duration, previousDuration);
Timeline<ShapeSet> phoneShapeSets = getShapeSets(*phone, duration, previousDuration);
// Result timing is relative to phone. Make absolute.
phoneShapeRules.shift(timedPhone.getStart());
phoneShapeSets.shift(timedPhone.getStart());
// Copy to timeline.
// Later shape rules may overwrite earlier ones if overlapping.
for (const auto& timedShapeRule : phoneShapeRules) {
shapeRules.set(timedShapeRule);
// Later shape sets may overwrite earlier ones if overlapping.
for (const auto& timedShapeSet : phoneShapeSets) {
shapeSets.set(timedShapeSet);
}
}
previousDuration = duration;
}
return shapeRules;
return shapeSets;
}
// Create timeline of shape rules using a bidirectional algorithm.
// Create timeline of shapes using a bidirectional algorithm.
// Here's a rough sketch:
//
// * Most consonants result in shape sets with multiple options; most vowels have only one shape option.
@ -187,13 +187,8 @@ JoiningContinuousTimeline<Shape> animate(const ContinuousTimeline<ShapeSet>& sha
}
JoiningContinuousTimeline<Shape> animate(const BoundedTimeline<Phone> &phones) {
// Create timeline of shape rules
ContinuousTimeline<ShapeRule> shapeRules = getShapeRules(phones);
// Take only the regular shapes from each shape rule. Alternative shapes will be implemented later.
ContinuousTimeline<ShapeSet> shapeSets(
shapeRules.getRange(), {{X}},
shapeRules | transformed([](const Timed<ShapeRule>& timedRule) { return Timed<ShapeSet>(timedRule.getTimeRange(), timedRule.getValue().regularShapes); }));
// Create timeline of shape sets
ContinuousTimeline<ShapeSet> shapeSets = getShapeSets(phones);
// Animate
JoiningContinuousTimeline<Shape> shapes = animate(shapeSets);