From a8df4ac4f525667c91c40132c87b6e69c43b7efe Mon Sep 17 00:00:00 2001 From: Daniel Wolf Date: Wed, 21 Dec 2016 22:30:38 +0100 Subject: [PATCH] Added --extendedShapes command-line parameter --- CMakeLists.txt | 3 +++ src/animation/animationRules.h | 4 ---- src/animation/mouthAnimation.cpp | 11 +++++++-- src/animation/mouthAnimation.h | 3 ++- src/animation/targetShapeSet.cpp | 38 ++++++++++++++++++++++++++++++++ src/animation/targetShapeSet.h | 16 ++++++++++++++ src/core/Shape.cpp | 23 +++++++++++++++++++ src/core/Shape.h | 11 ++++++++- src/exporters/Exporter.h | 2 +- src/exporters/JsonExporter.cpp | 4 ++-- src/exporters/JsonExporter.h | 2 +- src/exporters/TsvExporter.cpp | 5 +++-- src/exporters/TsvExporter.h | 2 +- src/exporters/XmlExporter.cpp | 4 ++-- src/exporters/XmlExporter.h | 2 +- src/exporters/exporterTools.cpp | 5 +++-- src/exporters/exporterTools.h | 2 +- src/lib/rhubarbLib.cpp | 6 +++-- src/lib/rhubarbLib.h | 3 +++ src/main.cpp | 18 ++++++++++++++- 20 files changed, 140 insertions(+), 24 deletions(-) create mode 100644 src/animation/targetShapeSet.cpp create mode 100644 src/animation/targetShapeSet.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 30b4318..6bdabc4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -229,6 +229,8 @@ add_library(rhubarb-animation src/animation/shapeRule.cpp src/animation/shapeRule.h src/animation/shapeShorthands.h + src/animation/targetShapeSet.cpp + src/animation/targetShapeSet.h src/animation/timingOptimization.cpp src/animation/timingOptimization.h src/animation/tweening.cpp @@ -298,6 +300,7 @@ add_library(rhubarb-exporters ) target_include_directories(rhubarb-exporters PUBLIC "src/exporters") target_link_libraries(rhubarb-exporters + rhubarb-animation rhubarb-core rhubarb-time ) diff --git a/src/animation/animationRules.h b/src/animation/animationRules.h index bd46245..c11ebab 100644 --- a/src/animation/animationRules.h +++ b/src/animation/animationRules.h @@ -11,10 +11,6 @@ Shape getBasicShape(Shape shape); // Returns the mouth shape that results from relaxing the specified shape. Shape relax(Shape shape); -// A set of mouth shapes that can be used to represent a certain sound. -// The actual selection will be performed based on similarity with the previous or next shape. -using ShapeSet = std::set; - // Gets the shape from a non-empty set of shapes that most closely resembles a reference shape. Shape getClosestShape(Shape reference, ShapeSet shapes); diff --git a/src/animation/mouthAnimation.cpp b/src/animation/mouthAnimation.cpp index f2ce096..f0387cf 100644 --- a/src/animation/mouthAnimation.cpp +++ b/src/animation/mouthAnimation.cpp @@ -5,16 +5,23 @@ #include "pauseAnimation.h" #include "tweening.h" #include "timingOptimization.h" +#include "targetShapeSet.h" -JoiningContinuousTimeline animate(const BoundedTimeline &phones) { +JoiningContinuousTimeline animate(const BoundedTimeline &phones, const ShapeSet& targetShapeSet) { // Create timeline of shape rules - const ContinuousTimeline shapeRules = getShapeRules(phones); + ContinuousTimeline shapeRules = getShapeRules(phones); + + // Modify shape rules to only contain allowed shapes -- plus X, which is needed for pauses and will be replaced later + ShapeSet targetShapeSetPlusX = targetShapeSet; + targetShapeSetPlusX.insert(Shape::X); + shapeRules = convertToTargetShapeSet(shapeRules, targetShapeSetPlusX); // Animate in multiple steps JoiningContinuousTimeline animation = animateRough(shapeRules); animation = optimizeTiming(animation); animation = animatePauses(animation); animation = insertTweens(animation); + animation = convertToTargetShapeSet(animation, targetShapeSet); for (const auto& timedShape : animation) { logTimedEvent("shape", timedShape); diff --git a/src/animation/mouthAnimation.h b/src/animation/mouthAnimation.h index aca06e7..07b0fce 100644 --- a/src/animation/mouthAnimation.h +++ b/src/animation/mouthAnimation.h @@ -3,5 +3,6 @@ #include "Phone.h" #include "Shape.h" #include "ContinuousTimeline.h" +#include "targetShapeSet.h" -JoiningContinuousTimeline animate(const BoundedTimeline& phones); +JoiningContinuousTimeline animate(const BoundedTimeline& phones, const ShapeSet& targetShapeSet); diff --git a/src/animation/targetShapeSet.cpp b/src/animation/targetShapeSet.cpp new file mode 100644 index 0000000..bb9fb33 --- /dev/null +++ b/src/animation/targetShapeSet.cpp @@ -0,0 +1,38 @@ +#include "targetShapeSet.h" + +Shape convertToTargetShapeSet(Shape shape, const ShapeSet& targetShapeSet) { + if (targetShapeSet.find(shape) != targetShapeSet.end()) { + return shape; + } + Shape basicShape = getBasicShape(shape); + if (targetShapeSet.find(basicShape) == targetShapeSet.end()) { + throw std::invalid_argument(fmt::format("Target shape set must contain basic shape {}.", basicShape)); + } + return basicShape; +} + +ShapeSet convertToTargetShapeSet(const ShapeSet& shapes, const ShapeSet& targetShapeSet) { + ShapeSet result; + for (Shape shape : shapes) { + result.insert(convertToTargetShapeSet(shape, targetShapeSet)); + } + return result; +} + +ContinuousTimeline convertToTargetShapeSet(const ContinuousTimeline& shapeRules, const ShapeSet& targetShapeSet) { + ContinuousTimeline result(shapeRules); + for (const auto& timedShapeRule : shapeRules) { + ShapeRule rule = timedShapeRule.getValue(); + std::get(rule) = convertToTargetShapeSet(std::get(rule), targetShapeSet); + result.set(timedShapeRule.getTimeRange(), rule); + } + return result; +} + +JoiningContinuousTimeline convertToTargetShapeSet(const JoiningContinuousTimeline& shapes, const ShapeSet& targetShapeSet) { + JoiningContinuousTimeline result(shapes); + for (const auto& timedShape : shapes) { + result.set(timedShape.getTimeRange(), convertToTargetShapeSet(timedShape.getValue(), targetShapeSet)); + } + return result; +} diff --git a/src/animation/targetShapeSet.h b/src/animation/targetShapeSet.h new file mode 100644 index 0000000..ef473f0 --- /dev/null +++ b/src/animation/targetShapeSet.h @@ -0,0 +1,16 @@ +#pragma once + +#include "Shape.h" +#include "shapeRule.h" + +// Returns the closest shape to the specified one that occurs in the target shape set. +Shape convertToTargetShapeSet(Shape shape, const ShapeSet& targetShapeSet); + +// Replaces each shape in the specified set with the closest shape that occurs in the target shape set. +ShapeSet convertToTargetShapeSet(const ShapeSet& shapes, const ShapeSet& targetShapeSet); + +// Replaces each shape in each rule with the closest shape that occurs in the target shape set. +ContinuousTimeline convertToTargetShapeSet(const ContinuousTimeline& shapeRules, const ShapeSet& targetShapeSet); + +// Replaces each shape in the specified timeline with the closest shape that occurs in the target shape set. +JoiningContinuousTimeline convertToTargetShapeSet(const JoiningContinuousTimeline& shapes, const ShapeSet& targetShapeSet); diff --git a/src/core/Shape.cpp b/src/core/Shape.cpp index 1b4975f..38c4390 100644 --- a/src/core/Shape.cpp +++ b/src/core/Shape.cpp @@ -1,12 +1,35 @@ #include "Shape.h" using std::string; +using std::set; ShapeConverter& ShapeConverter::get() { static ShapeConverter converter; return converter; } +set ShapeConverter::getBasicShapes() { + static const set result = [] { + set result; + for (int i = 0; i <= static_cast(Shape::LastBasicShape); ++i) { + result.insert(static_cast(i)); + } + return result; + }(); + return result; +} + +set ShapeConverter::getExtendedShapes() { + static const set result = [] { + set result; + for (int i = static_cast(Shape::LastBasicShape) + 1; i < static_cast(Shape::EndSentinel); ++i) { + result.insert(static_cast(i)); + } + return result; + }(); + return result; +} + string ShapeConverter::getTypeName() { return "Shape"; } diff --git a/src/core/Shape.h b/src/core/Shape.h index b211146..41f9246 100644 --- a/src/core/Shape.h +++ b/src/core/Shape.h @@ -1,6 +1,7 @@ #pragma once #include "EnumConverter.h" +#include // The classic Hanna-Barbera mouth shapes A-F phus the common supplements G-H // For reference, see http://sunewatts.dk/lipsync/lipsync/article_02.php @@ -14,6 +15,7 @@ enum class Shape { D, // Mouth wide open (vowels like f[a]ther, b[a]t, wh[y]) E, // Rounded mouth (vowels like [o]ff) F, // Puckered lips (y[ou], b[o]y, [w]ay) + LastBasicShape = F, // Extended shapes @@ -27,6 +29,8 @@ enum class Shape { class ShapeConverter : public EnumConverter { public: static ShapeConverter& get(); + std::set getBasicShapes(); + std::set getExtendedShapes(); protected: std::string getTypeName() override; member_data getMemberData() override; @@ -38,4 +42,9 @@ std::istream& operator>>(std::istream& stream, Shape& value); inline bool isClosed(Shape shape) { return shape == Shape::A || shape == Shape::X; -} \ No newline at end of file +} + +// A set of mouth shapes. +// This may be used to represent all shapes that can be used to represent a certain sound. +// Alternatively, it can represent all shapes the user wants to allow as program output. +using ShapeSet = std::set; diff --git a/src/exporters/Exporter.h b/src/exporters/Exporter.h index ef7e0ce..5c63479 100644 --- a/src/exporters/Exporter.h +++ b/src/exporters/Exporter.h @@ -7,5 +7,5 @@ class Exporter { public: virtual ~Exporter() {} - virtual void exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, std::ostream& outputStream) = 0; + virtual void exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, const ShapeSet& targetShapeSet, std::ostream& outputStream) = 0; }; diff --git a/src/exporters/JsonExporter.cpp b/src/exporters/JsonExporter.cpp index b0bde51..c1f5783 100644 --- a/src/exporters/JsonExporter.cpp +++ b/src/exporters/JsonExporter.cpp @@ -25,7 +25,7 @@ string escapeJsonString(const string& s) { return result; } -void JsonExporter::exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, std::ostream& outputStream) { +void JsonExporter::exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, const ShapeSet& targetShapeSet, std::ostream& outputStream) { // Export as JSON. // I'm not using a library because the code is short enough without one and it lets me control the formatting. outputStream << "{\n"; @@ -35,7 +35,7 @@ void JsonExporter::exportShapes(const boost::filesystem::path& inputFilePath, co outputStream << " },\n"; outputStream << " \"mouthCues\": [\n"; bool isFirst = true; - for (auto& timedShape : dummyShapeIfEmpty(shapes)) { + for (auto& timedShape : dummyShapeIfEmpty(shapes, targetShapeSet)) { if (!isFirst) outputStream << ",\n"; isFirst = false; outputStream << " { \"start\": " << formatDuration(timedShape.getStart()) diff --git a/src/exporters/JsonExporter.h b/src/exporters/JsonExporter.h index 112cb8b..ead85e4 100644 --- a/src/exporters/JsonExporter.h +++ b/src/exporters/JsonExporter.h @@ -4,5 +4,5 @@ class JsonExporter : public Exporter { public: - void exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, std::ostream& outputStream) override; + void exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, const ShapeSet& targetShapeSet, std::ostream& outputStream) override; }; diff --git a/src/exporters/TsvExporter.cpp b/src/exporters/TsvExporter.cpp index f4717b1..14d4ed7 100644 --- a/src/exporters/TsvExporter.cpp +++ b/src/exporters/TsvExporter.cpp @@ -1,6 +1,7 @@ #include "TsvExporter.h" +#include "targetShapeSet.h" -void TsvExporter::exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, std::ostream& outputStream) { +void TsvExporter::exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, const ShapeSet& targetShapeSet, std::ostream& outputStream) { UNUSED(inputFilePath); // Output shapes with start times @@ -9,5 +10,5 @@ void TsvExporter::exportShapes(const boost::filesystem::path& inputFilePath, con } // Output closed mouth with end time - outputStream << formatDuration(shapes.getRange().getEnd()) << "\t" << Shape::X << "\n"; + outputStream << formatDuration(shapes.getRange().getEnd()) << "\t" << convertToTargetShapeSet(Shape::X, targetShapeSet) << "\n"; } diff --git a/src/exporters/TsvExporter.h b/src/exporters/TsvExporter.h index 272d562..b950f4c 100644 --- a/src/exporters/TsvExporter.h +++ b/src/exporters/TsvExporter.h @@ -4,6 +4,6 @@ class TsvExporter : public Exporter { public: - void exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, std::ostream& outputStream) override; + void exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, const ShapeSet& targetShapeSet, std::ostream& outputStream) override; }; diff --git a/src/exporters/XmlExporter.cpp b/src/exporters/XmlExporter.cpp index c8cb8db..3f38c1d 100644 --- a/src/exporters/XmlExporter.cpp +++ b/src/exporters/XmlExporter.cpp @@ -6,7 +6,7 @@ using std::string; using boost::property_tree::ptree; -void XmlExporter::exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, std::ostream& outputStream) { +void XmlExporter::exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, const ShapeSet& targetShapeSet, std::ostream& outputStream) { ptree tree; // Add metadata @@ -14,7 +14,7 @@ void XmlExporter::exportShapes(const boost::filesystem::path& inputFilePath, con tree.put("rhubarbResult.metadata.duration", formatDuration(shapes.getRange().getDuration())); // Add mouth cues - for (auto& timedShape : dummyShapeIfEmpty(shapes)) { + for (auto& timedShape : dummyShapeIfEmpty(shapes, targetShapeSet)) { ptree& mouthCueElement = tree.add("rhubarbResult.mouthCues.mouthCue", timedShape.getValue()); mouthCueElement.put(".start", formatDuration(timedShape.getStart())); mouthCueElement.put(".end", formatDuration(timedShape.getEnd())); diff --git a/src/exporters/XmlExporter.h b/src/exporters/XmlExporter.h index 8257fb3..8495352 100644 --- a/src/exporters/XmlExporter.h +++ b/src/exporters/XmlExporter.h @@ -4,5 +4,5 @@ class XmlExporter : public Exporter { public: - void exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, std::ostream& outputStream) override; + void exportShapes(const boost::filesystem::path& inputFilePath, const JoiningContinuousTimeline& shapes, const ShapeSet& targetShapeSet, std::ostream& outputStream) override; }; diff --git a/src/exporters/exporterTools.cpp b/src/exporters/exporterTools.cpp index 5c2fcaf..f0e5c57 100644 --- a/src/exporters/exporterTools.cpp +++ b/src/exporters/exporterTools.cpp @@ -1,12 +1,13 @@ #include "exporterTools.h" +#include "targetShapeSet.h" // Makes sure there is at least one mouth shape -std::vector> dummyShapeIfEmpty(const JoiningTimeline& shapes) { +std::vector> dummyShapeIfEmpty(const JoiningTimeline& shapes, const ShapeSet& targetShapeSet) { std::vector> result; std::copy(shapes.begin(), shapes.end(), std::back_inserter(result)); if (result.empty()) { // Add zero-length empty mouth - result.push_back(Timed(0_cs, 0_cs, Shape::X)); + result.push_back(Timed(0_cs, 0_cs, convertToTargetShapeSet(Shape::X, targetShapeSet))); } return result; } diff --git a/src/exporters/exporterTools.h b/src/exporters/exporterTools.h index ef04203..d0b28fc 100644 --- a/src/exporters/exporterTools.h +++ b/src/exporters/exporterTools.h @@ -4,4 +4,4 @@ #include "Timeline.h" // Makes sure there is at least one mouth shape -std::vector> dummyShapeIfEmpty(const JoiningTimeline& shapes); +std::vector> dummyShapeIfEmpty(const JoiningTimeline& shapes, const ShapeSet& targetShapeSet); diff --git a/src/lib/rhubarbLib.cpp b/src/lib/rhubarbLib.cpp index 6ee8a3d..c643d15 100644 --- a/src/lib/rhubarbLib.cpp +++ b/src/lib/rhubarbLib.cpp @@ -13,11 +13,12 @@ using std::unique_ptr; JoiningContinuousTimeline animateAudioClip( const AudioClip& audioClip, optional dialog, + const ShapeSet& targetShapeSet, int maxThreadCount, ProgressSink& progressSink) { BoundedTimeline phones = recognizePhones(audioClip, dialog, maxThreadCount, progressSink); - JoiningContinuousTimeline result = animate(phones); + JoiningContinuousTimeline result = animate(phones, targetShapeSet); return result; } @@ -32,8 +33,9 @@ unique_ptr createWaveAudioClip(path filePath) { JoiningContinuousTimeline animateWaveFile( path filePath, optional dialog, + const ShapeSet& targetShapeSet, int maxThreadCount, ProgressSink& progressSink) { - return animateAudioClip(*createWaveAudioClip(filePath), dialog, maxThreadCount, progressSink); + return animateAudioClip(*createWaveAudioClip(filePath), dialog, targetShapeSet, maxThreadCount, progressSink); } diff --git a/src/lib/rhubarbLib.h b/src/lib/rhubarbLib.h index 97fcb94..a8d588f 100644 --- a/src/lib/rhubarbLib.h +++ b/src/lib/rhubarbLib.h @@ -5,15 +5,18 @@ #include "AudioClip.h" #include "ProgressBar.h" #include +#include "targetShapeSet.h" JoiningContinuousTimeline animateAudioClip( const AudioClip& audioClip, boost::optional dialog, + const ShapeSet& targetShapeSet, int maxThreadCount, ProgressSink& progressSink); JoiningContinuousTimeline animateWaveFile( boost::filesystem::path filePath, boost::optional dialog, + const ShapeSet& targetShapeSet, int maxThreadCount, ProgressSink& progressSink); diff --git a/src/main.cpp b/src/main.cpp index 510ce85..8b0f6f2 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -22,6 +22,7 @@ #include "JsonExporter.h" #include #include +#include "targetShapeSet.h" using std::exception; using std::string; @@ -81,6 +82,18 @@ unique_ptr createExporter(ExportFormat exportFormat) { } } +ShapeSet getTargetShapeSet(const string& extendedShapesString) { + // All basic shapes are mandatory + ShapeSet result(ShapeConverter::get().getBasicShapes()); + + // Add any extended shapes + for (char ch : extendedShapesString) { + Shape shape = ShapeConverter::get().parse(string(1, ch)); + result.insert(shape); + } + return result; +} + int main(int argc, char *argv[]) { auto pausableStderrSink = addPausableStdErrSink(logging::Level::Warn); pausableStderrSink->pause(); @@ -96,6 +109,7 @@ int main(int argc, char *argv[]) { tclap::ValueArg logFileName("", "logFile", "The log file path.", false, string(), "string", cmd); tclap::SwitchArg quietMode("q", "quiet", "Suppresses all output to stderr except for error messages.", cmd, false); tclap::ValueArg maxThreadCount("", "threads", "The maximum number of worker threads to use.", false, getProcessorCoreCount(), "number", cmd); + tclap::ValueArg extendedShapes("", "extendedShapes", "All extended, optional shapes to use.", false, "GHX", "string", cmd); tclap::ValueArg dialogFile("d", "dialogFile", "A file containing the text of the dialog.", false, string(), "string", cmd); auto exportFormats = vector(ExportFormatConverter::get().getValues()); tclap::ValuesConstraint exportFormatConstraint(exportFormats); @@ -120,6 +134,7 @@ int main(int argc, char *argv[]) { throw std::runtime_error("Thread count must be 1 or higher."); } path inputFilePath(inputFileName.getValue()); + ShapeSet targetShapeSet = getTargetShapeSet(extendedShapes.getValue()); // Set up log file if (logFileName.isSet()) { @@ -140,6 +155,7 @@ int main(int argc, char *argv[]) { animation = animateWaveFile( inputFilePath, dialogFile.isSet() ? readUtf8File(path(dialogFile.getValue())) : boost::optional(), + targetShapeSet, maxThreadCount.getValue(), progressBar); } @@ -147,7 +163,7 @@ int main(int argc, char *argv[]) { // Export animation unique_ptr exporter = createExporter(exportFormat.getValue()); - exporter->exportShapes(inputFilePath, animation, std::cout); + exporter->exportShapes(inputFilePath, animation, targetShapeSet, std::cout); logging::info("Exiting application normally."); } catch (...) {