jamosdev
/
rhubarb-lip-sync
mirror of https://github.com/DanielSWolf/rhubarb-lip-sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switched back to simple shape sets instead of shape rules 

The extra information can be generated automatically.
This commit is contained in:
Daniel Wolf 2016-12-08 11:37:48 +01:00
parent a24fe8874c
commit 21bea661c1
3 changed files with 58 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
src/animation/animationRules.cpp
View File

@ -71,45 +71,40 @@ optional<pair<Shape, TweenTiming>> getTween(Shape first, Shape second) {
return it != lookup.end() ? it->second : optional<pair<Shape, TweenTiming>>(); return it != lookup.end() ? it->second : optional<pair<Shape, TweenTiming>>();
} }
ShapeRule::ShapeRule(const ShapeSet& regularShapes, const ShapeSet& alternativeShapes) : Timeline<ShapeSet> getShapeSets(Phone phone, centiseconds duration, centiseconds previousDuration) {
regularShapes(regularShapes),
alternativeShapes(alternativeShapes)
{}
Timeline<ShapeRule> getShapeRules(Phone phone, centiseconds duration, centiseconds previousDuration) {
// Returns a timeline with a single shape set // Returns a timeline with a single shape set
auto single = [duration](ShapeRule value) { auto single = [duration](ShapeSet value) {
return Timeline<ShapeRule> {{0_cs, duration, value}}; return Timeline<ShapeSet> {{0_cs, duration, value}};
}; };
// Returns a timeline with two shape sets, timed as a diphthong // Returns a timeline with two shape sets, timed as a diphthong
auto diphthong = [duration](ShapeRule first, ShapeRule second) { auto diphthong = [duration](ShapeSet first, ShapeSet second) {
centiseconds firstDuration = duration_cast<centiseconds>(duration * 0.6); centiseconds firstDuration = duration_cast<centiseconds>(duration * 0.6);
return Timeline<ShapeRule> { return Timeline<ShapeSet> {
{0_cs, firstDuration, first}, {0_cs, firstDuration, first},
{firstDuration, duration, second} {firstDuration, duration, second}
}; };
}; };
// Returns a timeline with two shape sets, timed as a plosive // Returns a timeline with two shape sets, timed as a plosive
auto plosive = [duration, previousDuration](ShapeRule first, ShapeRule second) { auto plosive = [duration, previousDuration](ShapeSet first, ShapeSet second) {
centiseconds minOcclusionDuration = 4_cs; centiseconds minOcclusionDuration = 4_cs;
centiseconds maxOcclusionDuration = 12_cs; centiseconds maxOcclusionDuration = 12_cs;
centiseconds occlusionDuration = clamp(previousDuration / 2, minOcclusionDuration, maxOcclusionDuration); centiseconds occlusionDuration = clamp(previousDuration / 2, minOcclusionDuration, maxOcclusionDuration);
return Timeline<ShapeRule> { return Timeline<ShapeSet> {
{-occlusionDuration, 0_cs, first}, {-occlusionDuration, 0_cs, first},
{0_cs, duration, second} {0_cs, duration, second}
}; };
}; };
// Returns the result of `getShapeRules` when called with identical arguments // Returns the result of `getShapeSets` when called with identical arguments
// except for a different phone. // except for a different phone.
auto like = [duration, previousDuration](Phone referencePhone) { auto like = [duration, previousDuration](Phone referencePhone) {
return getShapeRules(referencePhone, duration, previousDuration); return getShapeSets(referencePhone, duration, previousDuration);
}; };
static const ShapeRule any{{A, B, C, D, E, F, G, H, X}}; static const ShapeSet any{A, B, C, D, E, F, G, H, X};
static const ShapeRule anyOpen{{B, C, D, E, F, G, H}}; static const ShapeSet anyOpen{B, C, D, E, F, G, H};
// Note: // Note:
// The shapes {A, B, G, X} are very similar. You should avoid regular shape sets containing more than one of these shapes. // The shapes {A, B, G, X} are very similar. You should avoid regular shape sets containing more than one of these shapes.
@ -117,52 +112,52 @@ Timeline<ShapeRule> getShapeRules(Phone phone, centiseconds duration, centisecon
// As an exception, a very flexible rule may contain *all* these shapes. // As an exception, a very flexible rule may contain *all* these shapes.
switch (phone) { switch (phone) {
case Phone::AO: return single({{E}}); case Phone::AO: return single({E});
case Phone::AA: return single({{D}}); case Phone::AA: return single({D});
case Phone::IY: return single({{B}, {C}}); case Phone::IY: return single({B});
case Phone::UW: return single({{F}}); case Phone::UW: return single({F});
case Phone::EH: return single({{C}}); case Phone::EH: return single({C});
case Phone::IH: return single({{B}, {C}}); case Phone::IH: return single({B});
case Phone::UH: return single({{F}}); case Phone::UH: return single({F});
case Phone::AH: return single({{C}}); case Phone::AH: return single({C});
case Phone::Schwa: return single({{B, C}}); case Phone::Schwa: return single({B, C});
case Phone::AE: return single({{C}}); case Phone::AE: return single({C});
case Phone::EY: return diphthong({{C}}, {{B}, {C}}); case Phone::EY: return diphthong({C}, {B});
case Phone::AY: return duration < 20_cs ? diphthong({{C}}, {{B}, {C}}) : diphthong({{D}}, {{B}, {C}}); case Phone::AY: return duration < 20_cs ? diphthong({C}, {B}) : diphthong({D}, {B});
case Phone::OW: return single({{F}}); case Phone::OW: return single({F});
case Phone::AW: return duration < 30_cs ? diphthong({{C}}, {{F}}) : diphthong({{D}}, {{F}}); case Phone::AW: return duration < 30_cs ? diphthong({C}, {F}) : diphthong({D}, {F});
case Phone::OY: return diphthong({{E}}, {{B}, {C}}); case Phone::OY: return diphthong({E}, {B});
case Phone::ER: return duration < 7_cs ? like(Phone::Schwa) : single({{B}}); case Phone::ER: return duration < 7_cs ? like(Phone::Schwa) : single({B});
case Phone::P: case Phone::P:
case Phone::B: return plosive({{A}}, any); case Phone::B: return plosive({A}, any);
case Phone::T: case Phone::T:
case Phone::D: return plosive({{B, F}}, anyOpen); case Phone::D: return plosive({B, F}, anyOpen);
case Phone::K: case Phone::K:
case Phone::G: return plosive({{B, C, E, F, H}}, anyOpen); case Phone::G: return plosive({B, C, E, F, H}, anyOpen);
case Phone::CH: case Phone::CH:
case Phone::JH: return single({{B, F}}); case Phone::JH: return single({B, F});
case Phone::F: case Phone::F:
case Phone::V: return single({{G}}); case Phone::V: return single({G});
case Phone::TH: case Phone::TH:
case Phone::DH: case Phone::DH:
case Phone::S: case Phone::S:
case Phone::Z: case Phone::Z:
case Phone::SH: case Phone::SH:
case Phone::ZH: return single({{B, F}}); case Phone::ZH: return single({B, F});
case Phone::HH: return single(any); // think "m-hm" case Phone::HH: return single(any); // think "m-hm"
case Phone::M: return single({{A}}); case Phone::M: return single({A});
case Phone::N: return single({{B, C, F, H}}); case Phone::N: return single({B, C, F, H});
case Phone::NG: return single({{B, C, E, F}}); case Phone::NG: return single({B, C, E, F});
case Phone::L: return duration < 20_cs ? single({{B, C, E, F, H}}) : single({{H}}); case Phone::L: return duration < 20_cs ? single({B, C, E, F, H}) : single({H});
case Phone::R: return single({{B, E, F}}); case Phone::R: return single({B, E, F});
case Phone::Y: return single({{B, C, F}}); case Phone::Y: return single({B, C, F});
case Phone::W: return single({{F}}); case Phone::W: return single({F});
case Phone::Breath: case Phone::Breath:
case Phone::Cough: case Phone::Cough:
case Phone::Smack: return single({{C}}); case Phone::Smack: return single({C});
case Phone::Noise: return single({{B}}); case Phone::Noise: return single({B});
default: throw std::invalid_argument("Unexpected phone."); default: throw std::invalid_argument("Unexpected phone.");
} }

29
src/animation/animationRules.h
View File

@ -14,7 +14,8 @@ Shape relax(Shape shape);
// Returns the mouth shape to use for *short* pauses between words. // Returns the mouth shape to use for *short* pauses between words.
Shape getRelaxedBridge(Shape lhs, Shape rhs); Shape getRelaxedBridge(Shape lhs, Shape rhs);
// A set of mouth shapes that can be used to represent a certain sound // A set of mouth shapes that can be used to represent a certain sound.
// The actual selection will be performed based on similarity with the previous or next shape.
using ShapeSet = std::set<Shape>; using ShapeSet = std::set<Shape>;
// Gets the shape from a non-empty set of shapes that most closely resembles a reference shape. // Gets the shape from a non-empty set of shapes that most closely resembles a reference shape.
@ -35,29 +36,7 @@ enum class TweenTiming {
// Returns the tween shape and timing to use to transition between the specified two mouth shapes. // Returns the tween shape and timing to use to transition between the specified two mouth shapes.
boost::optional<std::pair<Shape, TweenTiming>> getTween(Shape first, Shape second); boost::optional<std::pair<Shape, TweenTiming>> getTween(Shape first, Shape second);
// A struct describing the possible shapes to use during a given time range. // Returns the shape set(s) to use for a given phone.
struct ShapeRule {
ShapeRule(const ShapeSet& regularShapes, const ShapeSet& alternativeShapes = {});
// A set of one or more shapes that may be used to animate a given time range.
// The actual selection will be performed based on similarity with the previous or next shape.
ShapeSet regularShapes;
// The regular animation algorithm tries to minimize mouth shape changes. As a result, the mouth may sometimes remain static for too long.
// This is a set of zero or more shapes that may be used in these cases.
// In contrast to the regular shapes, this set should only contain shapes that can be used regardless of the surrounding shapes.
ShapeSet alternativeShapes;
bool operator==(const ShapeRule& rhs) const {
return regularShapes == rhs.regularShapes && alternativeShapes == rhs.alternativeShapes;
}
bool operator!=(const ShapeRule& rhs) const {
return !operator==(rhs);
}
};
// Returns the shape rule(s) to use for a given phone.
// The resulting timeline will always cover the entire duration of the phone (starting at 0 cs). // The resulting timeline will always cover the entire duration of the phone (starting at 0 cs).
// It may extend into the negative time range if animation is required prior to the sound being heard. // It may extend into the negative time range if animation is required prior to the sound being heard.
Timeline<ShapeRule> getShapeRules(Phone phone, centiseconds duration, centiseconds previousDuration); Timeline<ShapeSet> getShapeSets(Phone phone, centiseconds duration, centiseconds previousDuration);

29
src/animation/mouthAnimation.cpp
View File

@ -104,12 +104,12 @@ ContinuousTimeline<optional<T>, AutoJoin> boundedTimelinetoContinuousOptional(co
}; };
} }
ContinuousTimeline<ShapeRule> getShapeRules(const BoundedTimeline<Phone>& phones) { ContinuousTimeline<ShapeSet> getShapeSets(const BoundedTimeline<Phone>& phones) {
// Convert to continuous timeline so that silences aren't skipped when iterating // Convert to continuous timeline so that silences aren't skipped when iterating
auto continuousPhones = boundedTimelinetoContinuousOptional(phones); auto continuousPhones = boundedTimelinetoContinuousOptional(phones);
// Create timeline of shape rules // Create timeline of shape sets
ContinuousTimeline<ShapeRule> shapeRules(phones.getRange(), {{X}}); ContinuousTimeline<ShapeSet> shapeSets(phones.getRange(), {{X}});
centiseconds previousDuration = 0_cs; centiseconds previousDuration = 0_cs;
for (const auto& timedPhone : continuousPhones) { for (const auto& timedPhone : continuousPhones) {
optional<Phone> phone = timedPhone.getValue(); optional<Phone> phone = timedPhone.getValue();
@ -117,25 +117,25 @@ ContinuousTimeline<ShapeRule> getShapeRules(const BoundedTimeline<Phone>& phones
if (phone) { if (phone) {
// Animate one phone // Animate one phone
Timeline<ShapeRule> phoneShapeRules = getShapeRules(*phone, duration, previousDuration); Timeline<ShapeSet> phoneShapeSets = getShapeSets(*phone, duration, previousDuration);
// Result timing is relative to phone. Make absolute. // Result timing is relative to phone. Make absolute.
phoneShapeRules.shift(timedPhone.getStart()); phoneShapeSets.shift(timedPhone.getStart());
// Copy to timeline. // Copy to timeline.
// Later shape rules may overwrite earlier ones if overlapping. // Later shape sets may overwrite earlier ones if overlapping.
for (const auto& timedShapeRule : phoneShapeRules) { for (const auto& timedShapeSet : phoneShapeSets) {
shapeRules.set(timedShapeRule); shapeSets.set(timedShapeSet);
} }
} }
previousDuration = duration; previousDuration = duration;
} }
return shapeRules; return shapeSets;
} }
// Create timeline of shape rules using a bidirectional algorithm. // Create timeline of shapes using a bidirectional algorithm.
// Here's a rough sketch: // Here's a rough sketch:
// //
// * Most consonants result in shape sets with multiple options; most vowels have only one shape option. // * Most consonants result in shape sets with multiple options; most vowels have only one shape option.
@ -187,13 +187,8 @@ JoiningContinuousTimeline<Shape> animate(const ContinuousTimeline<ShapeSet>& sha
} }
JoiningContinuousTimeline<Shape> animate(const BoundedTimeline<Phone> &phones) { JoiningContinuousTimeline<Shape> animate(const BoundedTimeline<Phone> &phones) {
// Create timeline of shape rules // Create timeline of shape sets
ContinuousTimeline<ShapeRule> shapeRules = getShapeRules(phones); ContinuousTimeline<ShapeSet> shapeSets = getShapeSets(phones);
// Take only the regular shapes from each shape rule. Alternative shapes will be implemented later.
ContinuousTimeline<ShapeSet> shapeSets(
shapeRules.getRange(), {{X}},
shapeRules | transformed([](const Timed<ShapeRule>& timedRule) { return Timed<ShapeSet>(timedRule.getTimeRange(), timedRule.getValue().regularShapes); }));
// Animate // Animate
JoiningContinuousTimeline<Shape> shapes = animate(shapeSets); JoiningContinuousTimeline<Shape> shapes = animate(shapeSets);