Implemented round trip Spine - Rhubarb - Spine

This commit is contained in:
Daniel Wolf 2018-01-08 15:05:31 +01:00
parent 4d9ddf334f
commit 7bb1bec975
10 changed files with 420 additions and 58 deletions

View File

@ -1,11 +1,92 @@
package com.rhubarb_lip_sync.rhubarb_for_spine package com.rhubarb_lip_sync.rhubarb_for_spine
import javafx.beans.property.SimpleListProperty
import javafx.beans.property.SimpleObjectProperty
import javafx.beans.property.SimpleStringProperty
import javafx.collections.ObservableList
import java.nio.file.Path import java.nio.file.Path
import tornadofx.getValue
import tornadofx.observable
import tornadofx.setValue
import java.util.concurrent.Executors
class AnimationFileModel(animationFilePath: Path) { class AnimationFileModel(animationFilePath: Path) {
val spineJson: SpineJson val spineJson = SpineJson(animationFilePath)
val slotsProperty = SimpleObjectProperty<ObservableList<String>>()
var slots by slotsProperty
private set
val mouthSlotProperty: SimpleStringProperty = SimpleStringProperty().alsoListen {
mouthNaming = if (mouthSlot != null)
MouthNaming.guess(spineJson.getSlotAttachmentNames(mouthSlot))
else null
mouthShapes = if (mouthSlot != null) {
val mouthNames = spineJson.getSlotAttachmentNames(mouthSlot)
MouthShape.values().filter { mouthNames.contains(mouthNaming.getName(it)) }
} else null
mouthSlotError = if (mouthSlot != null) null
else "No slot with mouth drawings specified."
}
var mouthSlot by mouthSlotProperty
val mouthSlotErrorProperty = SimpleStringProperty()
var mouthSlotError by mouthSlotErrorProperty
private set
val mouthNamingProperty = SimpleObjectProperty<MouthNaming>()
var mouthNaming by mouthNamingProperty
private set
val mouthShapesProperty = SimpleObjectProperty<List<MouthShape>>().alsoListen {
mouthShapesError = getMouthShapesErrorString()
}
var mouthShapes by mouthShapesProperty
private set
val mouthShapesErrorProperty = SimpleStringProperty()
var mouthShapesError by mouthShapesErrorProperty
private set
val audioFileModelsProperty = SimpleListProperty<AudioFileModel>(
spineJson.audioEvents
.map { event ->
val executor = Executors.newSingleThreadExecutor()
AudioFileModel(event, this, executor, { result -> saveAnimation(result, event.name) })
}
.observable()
)
private fun saveAnimation(mouthCues: List<MouthCue>, audioEventName: String) {
val animationName = getAnimationName(audioEventName)
spineJson.createOrUpdateAnimation(mouthCues, audioEventName, animationName, mouthSlot, mouthNaming)
spineJson.save()
}
private fun getAnimationName(audioEventName: String): String = "say_$audioEventName"
val audioFileModels by audioFileModelsProperty
init { init {
spineJson = SpineJson(animationFilePath) slots = spineJson.slots.observable()
mouthSlot = spineJson.guessMouthSlot()
} }
private fun getMouthShapesErrorString(): String? {
val missingBasicShapes = MouthShape.basicShapes
.filter{ !mouthShapes.contains(it) }
if (missingBasicShapes.isEmpty()) return null
val result = StringBuilder()
result.append("Mouth shapes ${missingBasicShapes.joinToString()}")
result.appendln(if (missingBasicShapes.count() > 1) " are missing." else " is missing.")
val first = MouthShape.basicShapes.first()
val last = MouthShape.basicShapes.last()
result.append("At least the basic mouth shapes $first-$last need corresponding image attachments.")
return result.toString()
}
} }

View File

@ -0,0 +1,158 @@
package com.rhubarb_lip_sync.rhubarb_for_spine
import javafx.application.Platform
import javafx.beans.binding.ObjectBinding
import javafx.beans.binding.StringBinding
import javafx.beans.property.SimpleBooleanProperty
import javafx.beans.property.SimpleObjectProperty
import javafx.beans.property.SimpleStringProperty
import javafx.scene.control.Alert
import tornadofx.getValue
import tornadofx.setValue
import java.util.concurrent.ExecutorService
import java.util.concurrent.Future
class AudioFileModel(
audioEvent: SpineJson.AudioEvent,
private val parentModel: AnimationFileModel,
private val executor: ExecutorService,
private val reportResult: (List<MouthCue>) -> Unit
) {
val spineJson = parentModel.spineJson
val audioFilePath = spineJson.audioDirectoryPath.resolve(audioEvent.relativeAudioFilePath)
val eventNameProperty = SimpleStringProperty(audioEvent.name)
val eventName by eventNameProperty
val displayFilePathProperty = SimpleStringProperty(audioEvent.relativeAudioFilePath)
val displayFilePath by displayFilePathProperty
val dialogProperty = SimpleStringProperty(audioEvent.dialog)
val dialog: String? by dialogProperty
val animationProgressProperty = SimpleObjectProperty<Double?>(null)
var animationProgress by animationProgressProperty
private set
private val animatedPreviouslyProperty = SimpleBooleanProperty(false) // TODO: Initial value
private var animatedPreviously by animatedPreviouslyProperty
private val futureProperty = SimpleObjectProperty<Future<*>?>()
private var future by futureProperty
private val audioFileStateProperty = SimpleObjectProperty<AudioFileState>().apply {
bind(object : ObjectBinding<AudioFileState>() {
init {
super.bind(animatedPreviouslyProperty, futureProperty, animationProgressProperty)
}
override fun computeValue(): AudioFileState {
return if (future != null) {
if (animationProgress != null)
if (future!!.isCancelled)
AudioFileState(AudioFileStatus.Canceling)
else
AudioFileState(AudioFileStatus.Animating, animationProgress)
else
AudioFileState(AudioFileStatus.Pending)
} else {
if (animatedPreviously)
AudioFileState(AudioFileStatus.Done)
else
AudioFileState(AudioFileStatus.NotAnimated)
}
}
})
}
private val audioFileState by audioFileStateProperty
val statusLabelProperty = SimpleStringProperty().apply {
bind(object : StringBinding() {
init {
super.bind(audioFileStateProperty)
}
override fun computeValue(): String {
return when (audioFileState.status) {
AudioFileStatus.NotAnimated -> ""
AudioFileStatus.Pending -> "Waiting"
AudioFileStatus.Animating -> "${((animationProgress ?: 0.0) * 100).toInt()}%"
AudioFileStatus.Canceling -> "Canceling"
AudioFileStatus.Done -> "Done"
}
}
})
}
val statusLabel by statusLabelProperty
val actionLabelProperty = SimpleStringProperty().apply {
bind(object : StringBinding() {
init {
super.bind(futureProperty)
}
override fun computeValue(): String {
return if (future != null)
"Cancel"
else
"Animate"
}
})
}
val actionLabel by actionLabelProperty
fun performAction() {
if (future == null) {
startAnimation()
} else {
cancelAnimation()
}
}
private fun startAnimation() {
val wrapperTask = Runnable {
val extendedMouthShapes = parentModel.mouthShapes.filter { it.isExtended }.toSet()
val reportProgress: (Double?) -> Unit = {
progress -> runAndWait { this@AudioFileModel.animationProgress = progress }
}
val rhubarbTask = RhubarbTask(audioFilePath, dialog, extendedMouthShapes, reportProgress)
try {
try {
val result = rhubarbTask.call()
runAndWait {
reportResult(result)
animatedPreviously = true
}
} finally {
runAndWait {
animationProgress = null
future = null
}
}
} catch (e: InterruptedException) {
} catch (e: Exception) {
Platform.runLater {
Alert(Alert.AlertType.ERROR).apply {
headerText = "Error performing lip-sync for event $eventName."
contentText = e.toString()
show()
}
}
}
}
future = executor.submit(wrapperTask)
}
private fun cancelAnimation() {
future?.cancel(true)
}
}
enum class AudioFileStatus {
NotAnimated,
Pending,
Animating,
Canceling,
Done
}
data class AudioFileState(val status: AudioFileStatus, val progress: Double? = null)

View File

@ -10,8 +10,8 @@ import java.nio.file.InvalidPathException
import java.nio.file.Paths import java.nio.file.Paths
class MainModel { class MainModel {
val filePathStringProperty = SimpleStringProperty(getDefaultPathString()).applyListener { value -> val filePathStringProperty = SimpleStringProperty(getDefaultPathString()).alsoListen { value ->
try { filePathError = getExceptionMessage {
animationFileModel = null animationFileModel = null
if (value.isNullOrBlank()) { if (value.isNullOrBlank()) {
throw Exception("No input file specified.") throw Exception("No input file specified.")
@ -29,13 +29,8 @@ class MainModel {
} }
animationFileModel = AnimationFileModel(path) animationFileModel = AnimationFileModel(path)
filePathError = null
} catch (e: Exception) {
filePathError = e.message
} }
} }
var filePathString by filePathStringProperty var filePathString by filePathStringProperty
val filePathErrorProperty = SimpleStringProperty() val filePathErrorProperty = SimpleStringProperty()

View File

@ -1,10 +1,17 @@
package com.rhubarb_lip_sync.rhubarb_for_spine package com.rhubarb_lip_sync.rhubarb_for_spine
import javafx.beans.property.SimpleStringProperty
import javafx.event.ActionEvent
import javafx.event.EventHandler import javafx.event.EventHandler
import javafx.scene.control.Button
import javafx.scene.control.TableCell
import javafx.scene.control.TableView
import javafx.scene.control.TextField import javafx.scene.control.TextField
import javafx.scene.input.DragEvent import javafx.scene.input.DragEvent
import javafx.scene.input.TransferMode import javafx.scene.input.TransferMode
import javafx.stage.FileChooser
import tornadofx.* import tornadofx.*
import java.io.File
import java.time.LocalDate import java.time.LocalDate
import java.time.Period import java.time.Period
@ -16,47 +23,85 @@ class MainView : View() {
val age: Int get() = Period.between(birthday, LocalDate.now()).years val age: Int get() = Period.between(birthday, LocalDate.now()).years
} }
private val persons = listOf(
Person(1,"Samantha Stuart",LocalDate.of(1981,12,4)),
Person(2,"Tom Marks",LocalDate.of(2001,1,23)),
Person(3,"Stuart Gills",LocalDate.of(1989,5,23)),
Person(3,"Nicole Williams",LocalDate.of(1998,8,11))
).observable()
init { init {
title = "Rhubarb Lip Sync for Spine" title = "Rhubarb Lip Sync for Spine"
} }
override val root = form { override val root = form {
var filePathField: TextField? = null var filePathTextField: TextField? = null
var filePathButton: Button? = null
val fileModelProperty = mainModel.animationFileModelProperty
minWidth = 800.0 minWidth = 800.0
prefWidth = 1000.0
fieldset("Settings") { fieldset("Settings") {
field("Spine JSON file") { field("Spine JSON file") {
filePathField = textfield { filePathTextField = textfield {
textProperty().bindBidirectional(mainModel.filePathStringProperty) textProperty().bindBidirectional(mainModel.filePathStringProperty)
tooltip("Hello world")
errorProperty().bind(mainModel.filePathErrorProperty) errorProperty().bind(mainModel.filePathErrorProperty)
} }
button("...") filePathButton = button("...")
} }
field("Mouth slot") { field("Mouth slot") {
textfield() combobox<String> {
itemsProperty().bind(fileModelProperty.select { it!!.slotsProperty })
valueProperty().bindBidirectional(fileModelProperty.select { it!!.mouthSlotProperty })
errorProperty().bind(fileModelProperty.select { it!!.mouthSlotErrorProperty })
}
} }
field("Mouth naming") { field("Mouth naming") {
datepicker() label {
textProperty().bind(
fileModelProperty
.select { it!!.mouthNamingProperty }
.select { SimpleStringProperty(it.displayString) }
)
}
} }
field("Mouth shapes") { field("Mouth shapes") {
textfield() hbox {
label {
textProperty().bind(
fileModelProperty
.select { it!!.mouthShapesProperty }
.select {
val result = if (it.isEmpty()) "none" else it.joinToString()
SimpleStringProperty(result)
}
)
}
errorProperty().bind(fileModelProperty.select { it!!.mouthShapesErrorProperty })
}
} }
} }
fieldset("Audio events") { fieldset("Audio events") {
tableview(persons) { tableview<AudioFileModel> {
column("Event", Person::id) columnResizePolicy = TableView.CONSTRAINED_RESIZE_POLICY
column("Audio file", Person::name) column("Event", AudioFileModel::eventNameProperty)
column("Dialog", Person::birthday) column("Audio file", AudioFileModel::displayFilePathProperty)
column("Status", Person::age) column("Dialog", AudioFileModel::dialogProperty)
column("", Person::age) column("Status", AudioFileModel::statusLabelProperty)
column("", AudioFileModel::actionLabelProperty).apply {
setCellFactory { tableColumn ->
return@setCellFactory object : TableCell<AudioFileModel, String>() {
override fun updateItem(item: String?, empty: Boolean) {
super.updateItem(item, empty)
graphic = if (!empty)
Button(item).apply {
this.maxWidth = Double.MAX_VALUE
setOnAction {
val audioFileModel = this@tableview.items[index]
audioFileModel.performAction()
}
}
else
null
}
}
}
}
itemsProperty().bind(fileModelProperty.select { it!!.audioFileModelsProperty })
} }
} }
@ -68,10 +113,28 @@ class MainView : View() {
} }
onDragDropped = EventHandler<DragEvent> { event -> onDragDropped = EventHandler<DragEvent> { event ->
if (event.dragboard.hasFiles()) { if (event.dragboard.hasFiles()) {
filePathField!!.text = event.dragboard.files.firstOrNull()?.path filePathTextField!!.text = event.dragboard.files.firstOrNull()?.path
event.isDropCompleted = true event.isDropCompleted = true
event.consume() event.consume()
} }
} }
filePathButton!!.onAction = EventHandler<ActionEvent> {
val fileChooser = FileChooser().apply {
title = "Open Spine JSON file"
extensionFilters.addAll(
FileChooser.ExtensionFilter("Spine JSON file (*.json)", "*.json"),
FileChooser.ExtensionFilter("All files (*.*)", "*.*")
)
val lastDirectory = filePathTextField!!.text?.let { File(it).parentFile }
if (lastDirectory != null && lastDirectory.isDirectory) {
initialDirectory = lastDirectory
}
}
val file = fileChooser.showOpenDialog(this@MainView.primaryStage)
if (file != null) {
filePathTextField!!.text = file.path
}
}
} }
} }

View File

@ -6,13 +6,20 @@ class MouthNaming(val prefix: String, val suffix: String, val mouthShapeCasing:
companion object { companion object {
fun guess(mouthNames: List<String>): MouthNaming { fun guess(mouthNames: List<String>): MouthNaming {
if (mouthNames.isEmpty()) return MouthNaming("", "", guessMouthShapeCasing("")) if (mouthNames.isEmpty()) {
return MouthNaming("", "", guessMouthShapeCasing(""))
}
val commonPrefix = mouthNames.commonPrefix val commonPrefix = mouthNames.commonPrefix
val commonSuffix = mouthNames.commonSuffix val commonSuffix = mouthNames.commonSuffix
val shapeName = mouthNames.first().substring( val firstMouthName = mouthNames.first()
if (commonPrefix.length + commonSuffix.length >= firstMouthName.length) {
return MouthNaming(commonPrefix, "", guessMouthShapeCasing(""))
}
val shapeName = firstMouthName.substring(
commonPrefix.length, commonPrefix.length,
mouthNames.first().length - commonSuffix.length) firstMouthName.length - commonSuffix.length)
val mouthShapeCasing = guessMouthShapeCasing(shapeName) val mouthShapeCasing = guessMouthShapeCasing(shapeName)
return MouthNaming(commonPrefix, commonSuffix, mouthShapeCasing) return MouthNaming(commonPrefix, commonSuffix, mouthShapeCasing)
} }

View File

@ -1,5 +1,17 @@
package com.rhubarb_lip_sync.rhubarb_for_spine package com.rhubarb_lip_sync.rhubarb_for_spine
enum class MouthShape { enum class MouthShape {
A, B, C, D, E, F, G, H, X A, B, C, D, E, F, G, H, X;
}
val isBasic: Boolean
get() = this.ordinal < basicShapeCount
val isExtended: Boolean
get() = !this.isBasic
companion object {
val basicShapeCount = 6
val basicShapes = MouthShape.values().take(basicShapeCount)
}
}

View File

@ -1,6 +0,0 @@
package com.rhubarb_lip_sync.rhubarb_for_spine
// Modeled after C#'s IProgress<double>
interface Progress {
fun reportProgress(progress: Double)
}

View File

@ -6,10 +6,7 @@ import com.beust.klaxon.double
import com.beust.klaxon.string import com.beust.klaxon.string
import com.beust.klaxon.Parser as JsonParser import com.beust.klaxon.Parser as JsonParser
import org.apache.commons.lang3.SystemUtils.IS_OS_WINDOWS import org.apache.commons.lang3.SystemUtils.IS_OS_WINDOWS
import java.io.BufferedReader import java.io.*
import java.io.EOFException
import java.io.InputStreamReader
import java.io.StringReader
import java.nio.charset.StandardCharsets import java.nio.charset.StandardCharsets
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
@ -21,7 +18,7 @@ class RhubarbTask(
val audioFilePath: Path, val audioFilePath: Path,
val dialog: String?, val dialog: String?,
val extendedMouthShapes: Set<MouthShape>, val extendedMouthShapes: Set<MouthShape>,
val progress: Progress val reportProgress: (Double?) -> Unit
) : Callable<List<MouthCue>> { ) : Callable<List<MouthCue>> {
override fun call(): List<MouthCue> { override fun call(): List<MouthCue> {
@ -32,24 +29,25 @@ class RhubarbTask(
throw IllegalArgumentException("File '$audioFilePath' does not exist."); throw IllegalArgumentException("File '$audioFilePath' does not exist.");
} }
val dialogFile = if (dialog != null) TemporaryTextFile(dialog) else null val dialogFile = if (dialog != null) TemporaryTextFile(dialog) else null
dialogFile.use { val outputFile = TemporaryTextFile()
val processBuilder = ProcessBuilder(createProcessBuilderArgs(dialogFile?.filePath)) dialogFile.use { outputFile.use {
val processBuilder = ProcessBuilder(createProcessBuilderArgs(dialogFile?.filePath)).apply {
// See http://java-monitor.com/forum/showthread.php?t=4067
redirectOutput(outputFile.filePath.toFile())
}
val process: Process = processBuilder.start() val process: Process = processBuilder.start()
val stdout = BufferedReader(InputStreamReader(process.inputStream, StandardCharsets.UTF_8))
val stderr = BufferedReader(InputStreamReader(process.errorStream, StandardCharsets.UTF_8)) val stderr = BufferedReader(InputStreamReader(process.errorStream, StandardCharsets.UTF_8))
try { try {
while (true) { while (true) {
val line = stderr.interruptibleReadLine() val line = stderr.interruptibleReadLine()
val message = parseJsonObject(line) val message = parseJsonObject(line)
when (message.string("type")!!) { when (message.string("type")!!) {
"progress" -> { "progress" -> {
progress.reportProgress(message.double("value")!!)} reportProgress(message.double("value")!!)}
"success" -> { "success" -> {
progress.reportProgress(1.0) reportProgress(1.0)
val resultString = stdout.readText() val resultString = String(Files.readAllBytes(outputFile.filePath), StandardCharsets.UTF_8)
return parseRhubarbResult(resultString) return parseRhubarbResult(resultString)
} }
"failure" -> { "failure" -> {
@ -63,7 +61,7 @@ class RhubarbTask(
} catch (e: EOFException) { } catch (e: EOFException) {
throw Exception("Rhubarb terminated unexpectedly.") throw Exception("Rhubarb terminated unexpectedly.")
} }
} }}
throw Exception("An unexpected error occurred.") throw Exception("An unexpected error occurred.")
} }
@ -127,7 +125,7 @@ class RhubarbTask(
+ " Expected to find it in '$guiBinDirectory' or any directory above.") + " Expected to find it in '$guiBinDirectory' or any directory above.")
} }
private class TemporaryTextFile(val text: String) : AutoCloseable { private class TemporaryTextFile(text: String = "") : AutoCloseable {
val filePath: Path = Files.createTempFile(null, null).also { val filePath: Path = Files.createTempFile(null, null).also {
Files.write(it, text.toByteArray(StandardCharsets.UTF_8)) Files.write(it, text.toByteArray(StandardCharsets.UTF_8))
} }

View File

@ -1,6 +1,7 @@
package com.rhubarb_lip_sync.rhubarb_for_spine package com.rhubarb_lip_sync.rhubarb_for_spine
import com.beust.klaxon.* import com.beust.klaxon.*
import java.nio.charset.StandardCharsets
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
@ -67,7 +68,7 @@ class SpineJson(val filePath: Path) {
return slots.mapNotNull { it.string("name") } return slots.mapNotNull { it.string("name") }
} }
val presumedMouthSlot: String? get() { fun guessMouthSlot(): String? {
return slots.firstOrNull { it.contains("mouth", ignoreCase = true) } return slots.firstOrNull { it.contains("mouth", ignoreCase = true) }
?: slots.firstOrNull() ?: slots.firstOrNull()
} }
@ -138,4 +139,9 @@ class SpineJson(val filePath: Path) {
) )
} }
} }
fun save() {
var string = json.toJsonString(prettyPrint = true)
Files.write(filePath, listOf(string), StandardCharsets.UTF_8)
}
} }

View File

@ -2,6 +2,11 @@ package com.rhubarb_lip_sync.rhubarb_for_spine
import javafx.application.Platform import javafx.application.Platform
import javafx.beans.property.Property import javafx.beans.property.Property
import java.util.concurrent.ExecutionException
import java.util.concurrent.locks.Condition
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
val List<String>.commonPrefix: String get() { val List<String>.commonPrefix: String get() {
return if (isEmpty()) "" else this.reduce { result, string -> result.commonPrefixWith(string) } return if (isEmpty()) "" else this.reduce { result, string -> result.commonPrefixWith(string) }
@ -11,7 +16,7 @@ val List<String>.commonSuffix: String get() {
return if (isEmpty()) "" else this.reduce { result, string -> result.commonSuffixWith(string) } return if (isEmpty()) "" else this.reduce { result, string -> result.commonSuffixWith(string) }
} }
fun <TValue, TProperty : Property<TValue>> TProperty.applyListener(listener: (TValue) -> Unit) : TProperty { fun <TValue, TProperty : Property<TValue>> TProperty.alsoListen(listener: (TValue) -> Unit) : TProperty {
// Notify the listener of the initial value. // Notify the listener of the initial value.
// If we did this synchronously, the listener's state would have to be fully initialized the // If we did this synchronously, the listener's state would have to be fully initialized the
// moment this function is called. So calling this function during object initialization might // moment this function is called. So calling this function during object initialization might
@ -21,3 +26,46 @@ fun <TValue, TProperty : Property<TValue>> TProperty.applyListener(listener: (TV
addListener({ _, _, newValue -> listener(newValue)}) addListener({ _, _, newValue -> listener(newValue)})
return this return this
} }
fun getExceptionMessage(action: () -> Unit): String? {
try {
action();
} catch (e: Exception) {
return e.message
}
return null
}
/**
* Invokes a Runnable on the JFX thread and waits until it's finished.
* Similar to SwingUtilities.invokeAndWait.
* Based on http://www.guigarage.com/2013/01/invokeandwait-for-javafx/
*
* @throws InterruptedException Execution was interrupted
* @throws Throwable An exception occurred in the run method of the Runnable
*/
fun runAndWait(action: () -> Unit) {
if (Platform.isFxApplicationThread()) {
action()
} else {
val lock = ReentrantLock()
lock.withLock {
val doneCondition = lock.newCondition()
var throwable: Throwable? = null
Platform.runLater {
lock.withLock {
try {
action()
} catch (e: Throwable) {
throwable = e
} finally {
doneCondition.signal()
}
}
}
doneCondition.await()
throwable?.let { throw it }
}
}
}