Implemented round trip Spine - Rhubarb - Spine

This commit is contained in:
Daniel Wolf 2018-01-08 15:05:31 +01:00
parent 4d9ddf334f
commit 7bb1bec975
10 changed files with 420 additions and 58 deletions

View File

@ -1,11 +1,92 @@
package com.rhubarb_lip_sync.rhubarb_for_spine
import javafx.beans.property.SimpleListProperty
import javafx.beans.property.SimpleObjectProperty
import javafx.beans.property.SimpleStringProperty
import javafx.collections.ObservableList
import java.nio.file.Path
import tornadofx.getValue
import tornadofx.observable
import tornadofx.setValue
import java.util.concurrent.Executors
class AnimationFileModel(animationFilePath: Path) {
val spineJson: SpineJson
val spineJson = SpineJson(animationFilePath)
val slotsProperty = SimpleObjectProperty<ObservableList<String>>()
var slots by slotsProperty
private set
val mouthSlotProperty: SimpleStringProperty = SimpleStringProperty().alsoListen {
mouthNaming = if (mouthSlot != null)
MouthNaming.guess(spineJson.getSlotAttachmentNames(mouthSlot))
else null
mouthShapes = if (mouthSlot != null) {
val mouthNames = spineJson.getSlotAttachmentNames(mouthSlot)
MouthShape.values().filter { mouthNames.contains(mouthNaming.getName(it)) }
} else null
mouthSlotError = if (mouthSlot != null) null
else "No slot with mouth drawings specified."
}
var mouthSlot by mouthSlotProperty
val mouthSlotErrorProperty = SimpleStringProperty()
var mouthSlotError by mouthSlotErrorProperty
private set
val mouthNamingProperty = SimpleObjectProperty<MouthNaming>()
var mouthNaming by mouthNamingProperty
private set
val mouthShapesProperty = SimpleObjectProperty<List<MouthShape>>().alsoListen {
mouthShapesError = getMouthShapesErrorString()
}
var mouthShapes by mouthShapesProperty
private set
val mouthShapesErrorProperty = SimpleStringProperty()
var mouthShapesError by mouthShapesErrorProperty
private set
val audioFileModelsProperty = SimpleListProperty<AudioFileModel>(
spineJson.audioEvents
.map { event ->
val executor = Executors.newSingleThreadExecutor()
AudioFileModel(event, this, executor, { result -> saveAnimation(result, event.name) })
}
.observable()
)
private fun saveAnimation(mouthCues: List<MouthCue>, audioEventName: String) {
val animationName = getAnimationName(audioEventName)
spineJson.createOrUpdateAnimation(mouthCues, audioEventName, animationName, mouthSlot, mouthNaming)
spineJson.save()
}
private fun getAnimationName(audioEventName: String): String = "say_$audioEventName"
val audioFileModels by audioFileModelsProperty
init {
spineJson = SpineJson(animationFilePath)
slots = spineJson.slots.observable()
mouthSlot = spineJson.guessMouthSlot()
}
private fun getMouthShapesErrorString(): String? {
val missingBasicShapes = MouthShape.basicShapes
.filter{ !mouthShapes.contains(it) }
if (missingBasicShapes.isEmpty()) return null
val result = StringBuilder()
result.append("Mouth shapes ${missingBasicShapes.joinToString()}")
result.appendln(if (missingBasicShapes.count() > 1) " are missing." else " is missing.")
val first = MouthShape.basicShapes.first()
val last = MouthShape.basicShapes.last()
result.append("At least the basic mouth shapes $first-$last need corresponding image attachments.")
return result.toString()
}
}

View File

@ -0,0 +1,158 @@
package com.rhubarb_lip_sync.rhubarb_for_spine
import javafx.application.Platform
import javafx.beans.binding.ObjectBinding
import javafx.beans.binding.StringBinding
import javafx.beans.property.SimpleBooleanProperty
import javafx.beans.property.SimpleObjectProperty
import javafx.beans.property.SimpleStringProperty
import javafx.scene.control.Alert
import tornadofx.getValue
import tornadofx.setValue
import java.util.concurrent.ExecutorService
import java.util.concurrent.Future
class AudioFileModel(
audioEvent: SpineJson.AudioEvent,
private val parentModel: AnimationFileModel,
private val executor: ExecutorService,
private val reportResult: (List<MouthCue>) -> Unit
) {
val spineJson = parentModel.spineJson
val audioFilePath = spineJson.audioDirectoryPath.resolve(audioEvent.relativeAudioFilePath)
val eventNameProperty = SimpleStringProperty(audioEvent.name)
val eventName by eventNameProperty
val displayFilePathProperty = SimpleStringProperty(audioEvent.relativeAudioFilePath)
val displayFilePath by displayFilePathProperty
val dialogProperty = SimpleStringProperty(audioEvent.dialog)
val dialog: String? by dialogProperty
val animationProgressProperty = SimpleObjectProperty<Double?>(null)
var animationProgress by animationProgressProperty
private set
private val animatedPreviouslyProperty = SimpleBooleanProperty(false) // TODO: Initial value
private var animatedPreviously by animatedPreviouslyProperty
private val futureProperty = SimpleObjectProperty<Future<*>?>()
private var future by futureProperty
private val audioFileStateProperty = SimpleObjectProperty<AudioFileState>().apply {
bind(object : ObjectBinding<AudioFileState>() {
init {
super.bind(animatedPreviouslyProperty, futureProperty, animationProgressProperty)
}
override fun computeValue(): AudioFileState {
return if (future != null) {
if (animationProgress != null)
if (future!!.isCancelled)
AudioFileState(AudioFileStatus.Canceling)
else
AudioFileState(AudioFileStatus.Animating, animationProgress)
else
AudioFileState(AudioFileStatus.Pending)
} else {
if (animatedPreviously)
AudioFileState(AudioFileStatus.Done)
else
AudioFileState(AudioFileStatus.NotAnimated)
}
}
})
}
private val audioFileState by audioFileStateProperty
val statusLabelProperty = SimpleStringProperty().apply {
bind(object : StringBinding() {
init {
super.bind(audioFileStateProperty)
}
override fun computeValue(): String {
return when (audioFileState.status) {
AudioFileStatus.NotAnimated -> ""
AudioFileStatus.Pending -> "Waiting"
AudioFileStatus.Animating -> "${((animationProgress ?: 0.0) * 100).toInt()}%"
AudioFileStatus.Canceling -> "Canceling"
AudioFileStatus.Done -> "Done"
}
}
})
}
val statusLabel by statusLabelProperty
val actionLabelProperty = SimpleStringProperty().apply {
bind(object : StringBinding() {
init {
super.bind(futureProperty)
}
override fun computeValue(): String {
return if (future != null)
"Cancel"
else
"Animate"
}
})
}
val actionLabel by actionLabelProperty
fun performAction() {
if (future == null) {
startAnimation()
} else {
cancelAnimation()
}
}
private fun startAnimation() {
val wrapperTask = Runnable {
val extendedMouthShapes = parentModel.mouthShapes.filter { it.isExtended }.toSet()
val reportProgress: (Double?) -> Unit = {
progress -> runAndWait { this@AudioFileModel.animationProgress = progress }
}
val rhubarbTask = RhubarbTask(audioFilePath, dialog, extendedMouthShapes, reportProgress)
try {
try {
val result = rhubarbTask.call()
runAndWait {
reportResult(result)
animatedPreviously = true
}
} finally {
runAndWait {
animationProgress = null
future = null
}
}
} catch (e: InterruptedException) {
} catch (e: Exception) {
Platform.runLater {
Alert(Alert.AlertType.ERROR).apply {
headerText = "Error performing lip-sync for event $eventName."
contentText = e.toString()
show()
}
}
}
}
future = executor.submit(wrapperTask)
}
private fun cancelAnimation() {
future?.cancel(true)
}
}
enum class AudioFileStatus {
NotAnimated,
Pending,
Animating,
Canceling,
Done
}
data class AudioFileState(val status: AudioFileStatus, val progress: Double? = null)

View File

@ -10,8 +10,8 @@ import java.nio.file.InvalidPathException
import java.nio.file.Paths
class MainModel {
val filePathStringProperty = SimpleStringProperty(getDefaultPathString()).applyListener { value ->
try {
val filePathStringProperty = SimpleStringProperty(getDefaultPathString()).alsoListen { value ->
filePathError = getExceptionMessage {
animationFileModel = null
if (value.isNullOrBlank()) {
throw Exception("No input file specified.")
@ -29,13 +29,8 @@ class MainModel {
}
animationFileModel = AnimationFileModel(path)
filePathError = null
} catch (e: Exception) {
filePathError = e.message
}
}
var filePathString by filePathStringProperty
val filePathErrorProperty = SimpleStringProperty()

View File

@ -1,10 +1,17 @@
package com.rhubarb_lip_sync.rhubarb_for_spine
import javafx.beans.property.SimpleStringProperty
import javafx.event.ActionEvent
import javafx.event.EventHandler
import javafx.scene.control.Button
import javafx.scene.control.TableCell
import javafx.scene.control.TableView
import javafx.scene.control.TextField
import javafx.scene.input.DragEvent
import javafx.scene.input.TransferMode
import javafx.stage.FileChooser
import tornadofx.*
import java.io.File
import java.time.LocalDate
import java.time.Period
@ -16,47 +23,85 @@ class MainView : View() {
val age: Int get() = Period.between(birthday, LocalDate.now()).years
}
private val persons = listOf(
Person(1,"Samantha Stuart",LocalDate.of(1981,12,4)),
Person(2,"Tom Marks",LocalDate.of(2001,1,23)),
Person(3,"Stuart Gills",LocalDate.of(1989,5,23)),
Person(3,"Nicole Williams",LocalDate.of(1998,8,11))
).observable()
init {
title = "Rhubarb Lip Sync for Spine"
}
override val root = form {
var filePathField: TextField? = null
var filePathTextField: TextField? = null
var filePathButton: Button? = null
val fileModelProperty = mainModel.animationFileModelProperty
minWidth = 800.0
prefWidth = 1000.0
fieldset("Settings") {
field("Spine JSON file") {
filePathField = textfield {
filePathTextField = textfield {
textProperty().bindBidirectional(mainModel.filePathStringProperty)
tooltip("Hello world")
errorProperty().bind(mainModel.filePathErrorProperty)
}
button("...")
filePathButton = button("...")
}
field("Mouth slot") {
textfield()
combobox<String> {
itemsProperty().bind(fileModelProperty.select { it!!.slotsProperty })
valueProperty().bindBidirectional(fileModelProperty.select { it!!.mouthSlotProperty })
errorProperty().bind(fileModelProperty.select { it!!.mouthSlotErrorProperty })
}
}
field("Mouth naming") {
datepicker()
label {
textProperty().bind(
fileModelProperty
.select { it!!.mouthNamingProperty }
.select { SimpleStringProperty(it.displayString) }
)
}
}
field("Mouth shapes") {
textfield()
hbox {
label {
textProperty().bind(
fileModelProperty
.select { it!!.mouthShapesProperty }
.select {
val result = if (it.isEmpty()) "none" else it.joinToString()
SimpleStringProperty(result)
}
)
}
errorProperty().bind(fileModelProperty.select { it!!.mouthShapesErrorProperty })
}
}
}
fieldset("Audio events") {
tableview(persons) {
column("Event", Person::id)
column("Audio file", Person::name)
column("Dialog", Person::birthday)
column("Status", Person::age)
column("", Person::age)
tableview<AudioFileModel> {
columnResizePolicy = TableView.CONSTRAINED_RESIZE_POLICY
column("Event", AudioFileModel::eventNameProperty)
column("Audio file", AudioFileModel::displayFilePathProperty)
column("Dialog", AudioFileModel::dialogProperty)
column("Status", AudioFileModel::statusLabelProperty)
column("", AudioFileModel::actionLabelProperty).apply {
setCellFactory { tableColumn ->
return@setCellFactory object : TableCell<AudioFileModel, String>() {
override fun updateItem(item: String?, empty: Boolean) {
super.updateItem(item, empty)
graphic = if (!empty)
Button(item).apply {
this.maxWidth = Double.MAX_VALUE
setOnAction {
val audioFileModel = this@tableview.items[index]
audioFileModel.performAction()
}
}
else
null
}
}
}
}
itemsProperty().bind(fileModelProperty.select { it!!.audioFileModelsProperty })
}
}
@ -68,10 +113,28 @@ class MainView : View() {
}
onDragDropped = EventHandler<DragEvent> { event ->
if (event.dragboard.hasFiles()) {
filePathField!!.text = event.dragboard.files.firstOrNull()?.path
filePathTextField!!.text = event.dragboard.files.firstOrNull()?.path
event.isDropCompleted = true
event.consume()
}
}
filePathButton!!.onAction = EventHandler<ActionEvent> {
val fileChooser = FileChooser().apply {
title = "Open Spine JSON file"
extensionFilters.addAll(
FileChooser.ExtensionFilter("Spine JSON file (*.json)", "*.json"),
FileChooser.ExtensionFilter("All files (*.*)", "*.*")
)
val lastDirectory = filePathTextField!!.text?.let { File(it).parentFile }
if (lastDirectory != null && lastDirectory.isDirectory) {
initialDirectory = lastDirectory
}
}
val file = fileChooser.showOpenDialog(this@MainView.primaryStage)
if (file != null) {
filePathTextField!!.text = file.path
}
}
}
}

View File

@ -6,13 +6,20 @@ class MouthNaming(val prefix: String, val suffix: String, val mouthShapeCasing:
companion object {
fun guess(mouthNames: List<String>): MouthNaming {
if (mouthNames.isEmpty()) return MouthNaming("", "", guessMouthShapeCasing(""))
if (mouthNames.isEmpty()) {
return MouthNaming("", "", guessMouthShapeCasing(""))
}
val commonPrefix = mouthNames.commonPrefix
val commonSuffix = mouthNames.commonSuffix
val shapeName = mouthNames.first().substring(
val firstMouthName = mouthNames.first()
if (commonPrefix.length + commonSuffix.length >= firstMouthName.length) {
return MouthNaming(commonPrefix, "", guessMouthShapeCasing(""))
}
val shapeName = firstMouthName.substring(
commonPrefix.length,
mouthNames.first().length - commonSuffix.length)
firstMouthName.length - commonSuffix.length)
val mouthShapeCasing = guessMouthShapeCasing(shapeName)
return MouthNaming(commonPrefix, commonSuffix, mouthShapeCasing)
}

View File

@ -1,5 +1,17 @@
package com.rhubarb_lip_sync.rhubarb_for_spine
enum class MouthShape {
A, B, C, D, E, F, G, H, X
A, B, C, D, E, F, G, H, X;
val isBasic: Boolean
get() = this.ordinal < basicShapeCount
val isExtended: Boolean
get() = !this.isBasic
companion object {
val basicShapeCount = 6
val basicShapes = MouthShape.values().take(basicShapeCount)
}
}

View File

@ -1,6 +0,0 @@
package com.rhubarb_lip_sync.rhubarb_for_spine
// Modeled after C#'s IProgress<double>
interface Progress {
fun reportProgress(progress: Double)
}

View File

@ -6,10 +6,7 @@ import com.beust.klaxon.double
import com.beust.klaxon.string
import com.beust.klaxon.Parser as JsonParser
import org.apache.commons.lang3.SystemUtils.IS_OS_WINDOWS
import java.io.BufferedReader
import java.io.EOFException
import java.io.InputStreamReader
import java.io.StringReader
import java.io.*
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Path
@ -21,7 +18,7 @@ class RhubarbTask(
val audioFilePath: Path,
val dialog: String?,
val extendedMouthShapes: Set<MouthShape>,
val progress: Progress
val reportProgress: (Double?) -> Unit
) : Callable<List<MouthCue>> {
override fun call(): List<MouthCue> {
@ -32,24 +29,25 @@ class RhubarbTask(
throw IllegalArgumentException("File '$audioFilePath' does not exist.");
}
val dialogFile = if (dialog != null) TemporaryTextFile(dialog) else null
dialogFile.use {
val processBuilder = ProcessBuilder(createProcessBuilderArgs(dialogFile?.filePath))
val outputFile = TemporaryTextFile()
dialogFile.use { outputFile.use {
val processBuilder = ProcessBuilder(createProcessBuilderArgs(dialogFile?.filePath)).apply {
// See http://java-monitor.com/forum/showthread.php?t=4067
redirectOutput(outputFile.filePath.toFile())
}
val process: Process = processBuilder.start()
val stdout = BufferedReader(InputStreamReader(process.inputStream, StandardCharsets.UTF_8))
val stderr = BufferedReader(InputStreamReader(process.errorStream, StandardCharsets.UTF_8))
try {
while (true) {
val line = stderr.interruptibleReadLine()
val message = parseJsonObject(line)
when (message.string("type")!!) {
"progress" -> {
progress.reportProgress(message.double("value")!!)}
reportProgress(message.double("value")!!)}
"success" -> {
progress.reportProgress(1.0)
val resultString = stdout.readText()
reportProgress(1.0)
val resultString = String(Files.readAllBytes(outputFile.filePath), StandardCharsets.UTF_8)
return parseRhubarbResult(resultString)
}
"failure" -> {
@ -63,7 +61,7 @@ class RhubarbTask(
} catch (e: EOFException) {
throw Exception("Rhubarb terminated unexpectedly.")
}
}
}}
throw Exception("An unexpected error occurred.")
}
@ -127,7 +125,7 @@ class RhubarbTask(
+ " Expected to find it in '$guiBinDirectory' or any directory above.")
}
private class TemporaryTextFile(val text: String) : AutoCloseable {
private class TemporaryTextFile(text: String = "") : AutoCloseable {
val filePath: Path = Files.createTempFile(null, null).also {
Files.write(it, text.toByteArray(StandardCharsets.UTF_8))
}

View File

@ -1,6 +1,7 @@
package com.rhubarb_lip_sync.rhubarb_for_spine
import com.beust.klaxon.*
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Path
@ -67,7 +68,7 @@ class SpineJson(val filePath: Path) {
return slots.mapNotNull { it.string("name") }
}
val presumedMouthSlot: String? get() {
fun guessMouthSlot(): String? {
return slots.firstOrNull { it.contains("mouth", ignoreCase = true) }
?: slots.firstOrNull()
}
@ -138,4 +139,9 @@ class SpineJson(val filePath: Path) {
)
}
}
fun save() {
var string = json.toJsonString(prettyPrint = true)
Files.write(filePath, listOf(string), StandardCharsets.UTF_8)
}
}

View File

@ -2,6 +2,11 @@ package com.rhubarb_lip_sync.rhubarb_for_spine
import javafx.application.Platform
import javafx.beans.property.Property
import java.util.concurrent.ExecutionException
import java.util.concurrent.locks.Condition
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
val List<String>.commonPrefix: String get() {
return if (isEmpty()) "" else this.reduce { result, string -> result.commonPrefixWith(string) }
@ -11,7 +16,7 @@ val List<String>.commonSuffix: String get() {
return if (isEmpty()) "" else this.reduce { result, string -> result.commonSuffixWith(string) }
}
fun <TValue, TProperty : Property<TValue>> TProperty.applyListener(listener: (TValue) -> Unit) : TProperty {
fun <TValue, TProperty : Property<TValue>> TProperty.alsoListen(listener: (TValue) -> Unit) : TProperty {
// Notify the listener of the initial value.
// If we did this synchronously, the listener's state would have to be fully initialized the
// moment this function is called. So calling this function during object initialization might
@ -21,3 +26,46 @@ fun <TValue, TProperty : Property<TValue>> TProperty.applyListener(listener: (TV
addListener({ _, _, newValue -> listener(newValue)})
return this
}
fun getExceptionMessage(action: () -> Unit): String? {
try {
action();
} catch (e: Exception) {
return e.message
}
return null
}
/**
* Invokes a Runnable on the JFX thread and waits until it's finished.
* Similar to SwingUtilities.invokeAndWait.
* Based on http://www.guigarage.com/2013/01/invokeandwait-for-javafx/
*
* @throws InterruptedException Execution was interrupted
* @throws Throwable An exception occurred in the run method of the Runnable
*/
fun runAndWait(action: () -> Unit) {
if (Platform.isFxApplicationThread()) {
action()
} else {
val lock = ReentrantLock()
lock.withLock {
val doneCondition = lock.newCondition()
var throwable: Throwable? = null
Platform.runLater {
lock.withLock {
try {
action()
} catch (e: Throwable) {
throwable = e
} finally {
doneCondition.signal()
}
}
}
doneCondition.await()
throwable?.let { throw it }
}
}
}