Detecting individual phonemes

This commit is contained in:
Daniel Wolf 2015-09-10 21:31:25 +02:00
parent eb675e4968
commit f0285c6488
3 changed files with 194 additions and 6 deletions

View File

@ -3,7 +3,127 @@
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/CMakeLists.txt" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/acmod.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/acmod.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/allphone_search.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/allphone_search.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/bin_mdef.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/bin_mdef.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/blkarray_list.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/blkarray_list.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/dict.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/dict.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/dict2pid.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/dict2pid.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/fsg_history.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/fsg_history.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/fsg_lextree.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/fsg_lextree.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/fsg_search.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/hmm.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/hmm.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/kws_detections.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/kws_detections.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/kws_search.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/kws_search.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/mdef.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/mdef.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ms_gauden.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ms_gauden.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ms_mgau.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ms_mgau.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ms_senone.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ms_senone.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ngram_search.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ngram_search.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ngram_search_fwdflat.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ngram_search_fwdflat.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ngram_search_fwdtree.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ngram_search_fwdtree.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/phone_loop_search.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/phone_loop_search.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/pocketsphinx.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ps_alignment.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ps_alignment.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ps_lattice.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ps_mllr.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ptm_mgau.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/ptm_mgau.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/s2_semi_mgau.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/s2_semi_mgau.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/state_align_search.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/state_align_search.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/tmat.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/tmat.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/vector.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/pocketsphinx/src/libpocketsphinx/vector.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_interface.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_noise.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_noise.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_prespch_buf.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_prespch_buf.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_sigproc.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_warp.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_warp.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_warp_affine.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_warp_affine.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_warp_inverse_linear.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_warp_inverse_linear.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_warp_piecewise_linear.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fe_warp_piecewise_linear.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/fixlog.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/fe/yin.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/feat/agc.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/feat/cmn.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/feat/cmn_prior.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/feat/feat.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/feat/lda.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/fsg_model.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/jsgf.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/jsgf_parser.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/jsgf_parser.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/jsgf_scanner.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/jsgf_scanner.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/lm_trie.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/lm_trie.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/lm_trie_quant.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/lm_trie_quant.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/ngram_model.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/ngram_model_set.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/ngram_model_set.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/ngram_model_trie.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/ngram_model_trie.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/ngrams_raw.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/lm/ngrams_raw.h" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/bio.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/bitarr.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/bitvec.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/blas_lite.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/case.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/ckd_alloc.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/cmd_ln.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/dtoa.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/err.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/f2c_lite.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/filename.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/genrand.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/glist.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/hash_table.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/heap.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/huff_code.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/listelem_alloc.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/logmath.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/matrix.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/mmio.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/pio.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/priority_queue.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/profile.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/sbthread.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/slamch.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/slapack_lite.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lib/sphinxbase/src/libsphinxbase/util/strfuncs.c" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/main.cpp" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/tmp.cpp" isTestSource="false" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
@ -15,6 +135,8 @@
<root url="file://C:/MinGW/lib/gcc/mingw32/4.8.1/include-fixed" />
<root url="file://C:/MinGW/lib/gcc/mingw32/4.8.1/include/c++" />
<root url="file://C:/MinGW/lib/gcc/mingw32/4.8.1/include/ssp" />
<root url="file://$MODULE_DIR$/lib/pocketsphinx/include" />
<root url="file://$MODULE_DIR$/lib/sphinxbase/include" />
</CLASSES>
<SOURCES>
<root url="file://C:/MinGW/include" />
@ -22,6 +144,8 @@
<root url="file://C:/MinGW/lib/gcc/mingw32/4.8.1/include-fixed" />
<root url="file://C:/MinGW/lib/gcc/mingw32/4.8.1/include/c++" />
<root url="file://C:/MinGW/lib/gcc/mingw32/4.8.1/include/ssp" />
<root url="file://$MODULE_DIR$/lib/pocketsphinx/include" />
<root url="file://$MODULE_DIR$/lib/sphinxbase/include" />
</SOURCES>
</library>
</orderEntry>

View File

@ -4,4 +4,9 @@ project(LipSync)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
set(SOURCE_FILES main.cpp)
add_executable(LipSync ${SOURCE_FILES})
include_directories("lib/sphinxbase-5prealpha-2015-08-05/include" "lib/pocketsphinx-5prealpha-2015-08-05/include")
FILE(GLOB_RECURSE SPHINX_BASE "lib/sphinxbase-5prealpha-2015-08-05/src/libsphinxbase/*.c")
FILE(GLOB POCKETSPHINX "lib/pocketsphinx-5prealpha-2015-08-05/src/libpocketsphinx/*.c")
add_executable(LipSync ${SOURCE_FILES} ${SPHINX_BASE} ${POCKETSPHINX})

View File

@ -1,8 +1,67 @@
#include <iostream>
#include <pocketsphinx.h>
#include <stdexcept>
#include <fstream>
#include <memory>
using namespace std;
using std::runtime_error;
using std::shared_ptr;
#define MODELDIR "X:/dev/projects/LipSync/lib/pocketsphinx/model"
int main(int argc, char *argv[]) {
shared_ptr<cmd_ln_t> config(
cmd_ln_init(
nullptr, ps_args(), true,
// Set acoustic model
"-hmm", MODELDIR "/en-us/en-us",
// Set phonetic language model
"-allphone", MODELDIR "/en-us/en-us-phone.lm.bin",
// The following settings are Voodoo to me.
// I copied them from http://cmusphinx.sourceforge.net/wiki/phonemerecognition
// Set beam width applied to every frame in Viterbi search
"-beam", "1e-20",
// Set beam width applied to phone transitions
"-pbeam", "1e-20",
// Set language model probability weight
"-lw", "2.0",
nullptr),
[](cmd_ln_t* config) { cmd_ln_free_r(config); });
if (!config) throw runtime_error("Error creating configuration.");
shared_ptr<ps_decoder_t> recognizer(
ps_init(config.get()),
[](ps_decoder_t* recognizer) { ps_free(recognizer); });
if (!recognizer) throw runtime_error("Error creating speech recognizer.");
shared_ptr<FILE> file(
fopen("X:/dev/projects/LipSync/lib/pocketsphinx/test/data/goforward.raw", "rb"),
[](FILE* file) { fclose(file); });
if (!file) throw runtime_error("Error opening sound file.");
int error = ps_start_utt(recognizer.get());
if (error) throw runtime_error("Error starting utterance processing.");
int16 buffer[512];
while (!feof(file.get())) {
size_t sampleCount = fread(buffer, 2, 512, file.get());
int searchedFrameCount = ps_process_raw(recognizer.get(), buffer, sampleCount, false, false);
if (searchedFrameCount < 0) throw runtime_error("Error decoding raw audio data.");
}
error = ps_end_utt(recognizer.get());
if (error) throw runtime_error("Error ending utterance processing.");
ps_seg_t *segmentationIter;
int32 score;
for (segmentationIter = ps_seg_iter(recognizer.get(), &score); segmentationIter; segmentationIter = ps_seg_next(segmentationIter)) {
// Get phoneme
char const *phoneme = ps_seg_word(segmentationIter);
// Get timing
int startFrame, endFrame;
ps_seg_frames(segmentationIter, &startFrame, &endFrame);
printf(">>> %-5s %-5d %-5d\n", phoneme, startFrame, endFrame);
}
int main() {
cout << "Hello, World!" << endl;
return 0;
}