diff --git a/rhubarb/src/recognition/PocketSphinxRecognizer.cpp b/rhubarb/src/recognition/PocketSphinxRecognizer.cpp
index 24f0741..ca41050 100644
--- a/rhubarb/src/recognition/PocketSphinxRecognizer.cpp
+++ b/rhubarb/src/recognition/PocketSphinxRecognizer.cpp
@@ -154,6 +154,8 @@ optional<Timeline<Phone>> getPhoneAlignment(
 	const vector<int16_t>& audioBuffer,
 	ps_decoder_t& decoder)
 {
+	if (wordIds.empty()) return boost::none;
+
 	// Create alignment list
 	lambda_unique_ptr<ps_alignment_t> alignment(
 		ps_alignment_init(decoder.d2p),
@@ -300,7 +302,6 @@ static Timeline<Phone> utteranceToPhones(
 		const string fixedWord = fixPronunciation(timedWord.getValue());
 		wordIds.push_back(getWordId(fixedWord, *decoder.dict));
 	}
-	if (wordIds.empty()) return {};
 
 	// Align the words' phones with speech
 #if BOOST_VERSION < 105600 // Support legacy syntax