Adjust gitignore for vs

.gitignore

@@ -4,3 +4,9 @@ export
 addons/**/~*.dll
 .sconsign.dblite
 *.obj
+
+# Visual-studio related
+.vs
+*.sln
+*.vcxproj*
+x64

README.md

@@ -4,32 +4,41 @@ A GDExtension that adds to Godot4 support for encoding voice data using the Opus
 
 ## Overview
 
-This extension adds a new singleton to Godot: `Opus` with two methods: `encode` and `decode`.
+This extension adds a new node to Godot: `Opus` with three methods: `encode`, `decode` and `decode_and_play`.
 
 These can be used to compress audio obtained `AudioEffectCapture` and then to decode it so it's usable in Godot again.
 
-Quick and dirty example (full demo coming soon):
+Usage is best illustrated in this demo: https://github.com/microtaur/godot4-p2p-voip
+
+Most interesting part from the demo code illustrating how this works:
 
 ```GDScript
 
-func _process_audio() -> void:
-    if has_data() and active:
-        call_deferred("rpc", "play_data", get_data())
+var _encoder := Opus.new()
+
+...
+
+func _audio_process():
+	while true:
+		if has_data() and active:
+			var data = get_data()
+			call_deferred("rpc", "play_data", data)
+		else:
+			OS.delay_msec(10)
 
 ...
 
 func get_data() -> PackedFloat32Array:
-    var data = effect.get_buffer(BUFFER_SIZE)
-    return Opus.encode(data)
+	var data = effect.get_buffer(BUFFER_SIZE)
+	return _encoder.encode(data)
 
 ...
 
-@rpc("any_peer", "call_remote", "unreliable_ordered")
+@rpc("any_peer", "call_remote", "reliable")
 func play_data(data: PackedFloat32Array) -> void:
-    var id = client.multiplayer.get_remote_sender_id()
-    var decoded = Opus.decode(data)
-    for b in range(0, BUFFER_SIZE):
-        _get_generator(id).push_frame(decoded[b])
+	var id = client.multiplayer.get_remote_sender_id()
+	_update_player_pool()
+	_get_opus_instance(id).decode_and_play(_get_generator(id), data)
 
 ```
 

SConstruct

@@ -1,24 +1,29 @@
 #!/usr/bin/env python
 from glob import glob
 from pathlib import Path
-from SCons.Script import MSVSProject
+from SCons.Script import *
 
+# Initialize environment and SCons script for godot-cpp
 env = SConscript("godot-cpp/SConstruct")
+env.Tool('msvs')
 
-# Sources
+# Sources and include paths
 env.Append(CPPPATH=["src/"])
 sources = Glob("src/*.cpp")
+sources += Glob("src/platform/win32/*.cpp")
 
-# Opus (Windows x64)
-env.Append(CPPPATH=['#3rdparty/opus/include'])
-env.Append(LIBPATH=['#3rdparty/opus/lib'])
-env.Append(LIBS=['opus'])
+# Append additional library paths and libraries for Opus, Speex, and vpx
+env.Append(CPPPATH=['#3rdparty/opus/include', '#3rdparty/speex/include', '#3rdparty/libvpx/include'])
+env.Append(LIBPATH=['#3rdparty/opus/lib', '#3rdparty/speex/lib', '#3rdparty/libvpx/lib/x64'])
+env.Append(LIBS=['opus', 'libspeex', 'libspeexdsp', 'vpx'])
 
+# Determine extension and addon path
 (extension_path,) = glob("export/addons/*/*.gdextension")
 addon_path = Path(extension_path).parent
 project_name = Path(extension_path).stem
 debug_or_release = "release" if env["target"] == "template_release" else "debug"
 
+# Generate library based on platform
 if env["platform"] == "macos":
     library = env.SharedLibrary(
         "{0}/lib/lib{1}.{2}.{3}.framework/{1}.{2}.{3}".format(
@@ -42,4 +47,19 @@ else:
         source=sources,
     )
 
-Default(library)
+srcs = []
+for s in sources:
+    srcs.append(s.abspath)
+
+
+msvs_project = env.MSVSProject(
+    target = project_name + env['MSVSPROJECTSUFFIX'],
+    srcs = srcs,
+    include_dirs = env['CPPPATH'],
+    lib_dirs = env['LIBPATH'],
+    libs = env['LIBS'],
+    variant = 'Debug|x64',
+)
+
+# Make sure the default build includes the Visual Studio project
+Default(library, msvs_project)

src/AudioProcessor.cpp

@@ -0,0 +1,122 @@
+#include "AudioProcessor.h"
+
+#include <godot_cpp/core/class_db.hpp>
+#include <godot_cpp/variant/utility_functions.hpp>
+
+#pragma comment(lib, "ole32.lib")
+#pragma comment(lib, "Mmdevapi.lib")
+#pragma comment(lib, "Uuid.lib")
+
+namespace godot {
+
+void AudioProcessor::_bind_methods()
+{
+	ClassDB::bind_method(D_METHOD("process"), &AudioProcessor::process);
+	ClassDB::bind_method(D_METHOD("get_output_mix_rate"), &AudioProcessor::get_output_mix_rate);
+	ClassDB::bind_method(D_METHOD("get_input_mix_rate"), &AudioProcessor::get_input_mix_rate);
+}
+
+AudioProcessor::AudioProcessor()
+{
+  HRESULT hr;
+
+  CoInitialize(NULL);
+  hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), NULL, CLSCTX_ALL, __uuidof(IMMDeviceEnumerator), (void**)&m_deviceEnumerator);
+  if (FAILED(hr)) {
+    std::cerr << "Failed to create device enumerator" << std::endl;
+    return;
+  }
+
+  // Initialize default output device
+  hr = m_deviceEnumerator->GetDefaultAudioEndpoint(eRender, eConsole, &m_defaultOutputDevice);
+  if (FAILED(hr)) {
+    std::cerr << "Failed to get default output audio device" << std::endl;
+    return;
+  }
+
+  hr = m_defaultOutputDevice->Activate(__uuidof(IAudioClient), CLSCTX_ALL, NULL, (void**)&m_outputAudioClient);
+  if (FAILED(hr)) {
+    std::cerr << "Failed to activate output audio client" << std::endl;
+    return;
+  }
+
+  hr = m_outputAudioClient->GetMixFormat(&m_outputMixFormat);
+  if (FAILED(hr)) {
+    std::cerr << "Failed to get output mix format" << std::endl;
+    return;
+  }
+
+  // Initialize default input device
+  hr = m_deviceEnumerator->GetDefaultAudioEndpoint(eCapture, eConsole, &m_defaultInputDevice);
+  if (FAILED(hr)) {
+    std::cerr << "Failed to get default input audio device" << std::endl;
+    return;
+  }
+
+  hr = m_defaultInputDevice->Activate(__uuidof(IAudioClient), CLSCTX_ALL, NULL, (void**)&m_inputAudioClient);
+  if (FAILED(hr)) {
+    std::cerr << "Failed to activate input audio client" << std::endl;
+    return;
+  }
+
+  hr = m_inputAudioClient->GetMixFormat(&m_inputMixFormat);
+  if (FAILED(hr)) {
+    std::cerr << "Failed to get input mix format" << std::endl;
+    return;
+  }
+}
+
+AudioProcessor::~AudioProcessor() {
+  if (m_outputMixFormat != nullptr) {
+    CoTaskMemFree(m_outputMixFormat);
+    m_outputMixFormat = nullptr;
+  }
+
+  if (m_outputAudioClient != nullptr) {
+    m_outputAudioClient->Release();
+    m_outputAudioClient = nullptr;
+  }
+
+  if (m_defaultOutputDevice != nullptr) {
+    m_defaultOutputDevice->Release();
+    m_defaultOutputDevice = nullptr;
+  }
+
+  if (m_inputMixFormat != nullptr) {
+    CoTaskMemFree(m_inputMixFormat);
+    m_inputMixFormat = nullptr;
+  }
+
+  if (m_inputAudioClient != nullptr) {
+    m_inputAudioClient->Release();
+    m_inputAudioClient = nullptr;
+  }
+
+  if (m_defaultInputDevice != nullptr) {
+    m_defaultInputDevice->Release();
+    m_defaultInputDevice = nullptr;
+  }
+
+  if (m_deviceEnumerator != nullptr) {
+    m_deviceEnumerator->Release();
+    m_deviceEnumerator = nullptr;
+  }
+
+  CoUninitialize();
+}
+
+void AudioProcessor::process()
+{
+
+}
+
+size_t AudioProcessor::get_output_mix_rate() const
+{
+	return m_outputMixFormat->nSamplesPerSec;
+}
+
+size_t AudioProcessor::get_input_mix_rate() const {
+  return m_inputMixFormat->nSamplesPerSec;
+}
+
+}

src/AudioProcessor.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include <godot_cpp/classes/node.hpp>
+#include <godot_cpp/core/class_db.hpp>
+
+#include <mmdeviceapi.h>
+#include <Audioclient.h>
+
+#include "opus.h"
+
+namespace godot {
+
+class AudioProcessor : public Node
+{
+	GDCLASS(AudioProcessor, Node);
+
+protected:
+	static void _bind_methods();
+
+public:
+	AudioProcessor();
+	~AudioProcessor();
+
+	void process();
+
+	size_t get_input_mix_rate() const;
+	size_t get_output_mix_rate() const;
+
+private:
+	IMMDeviceEnumerator* m_deviceEnumerator = nullptr;
+	IMMDevice* m_defaultOutputDevice = nullptr;
+	IAudioClient* m_outputAudioClient = nullptr;
+	WAVEFORMATEX* m_outputMixFormat = nullptr;
+
+	IMMDevice* m_defaultInputDevice = nullptr;
+	IAudioClient* m_inputAudioClient = nullptr;
+	WAVEFORMATEX* m_inputMixFormat = nullptr;
+};
+
+}

src/GodotOpus.cpp

@@ -7,6 +7,7 @@ namespace godot {
 
 void Opus::_bind_methods()
 {
+	ClassDB::bind_method(D_METHOD("update_mix_rate"), &Opus::update_mix_rate);
 	ClassDB::bind_method(D_METHOD("encode"), &Opus::encode);
 	ClassDB::bind_method(D_METHOD("decode"), &Opus::decode);
 	ClassDB::bind_method(D_METHOD("decode_and_play"), &Opus::decode_and_play);
@@ -16,11 +17,12 @@ Opus::Opus()
 {
 	int err{};
 
-	m_encoder = opus_encoder_create(48000, 1, OPUS_APPLICATION_VOIP, &err);
+	// Opus
+	m_encoder = opus_encoder_create(48000, 2, OPUS_APPLICATION_VOIP, &err);
 	ERR_FAIL_COND(err != OPUS_OK);
 	ERR_FAIL_COND(m_encoder == nullptr);
 
-	m_decoder = opus_decoder_create(48000, 1, &err);
+	m_decoder = opus_decoder_create(48000, 2, &err);
 	ERR_FAIL_COND(err != OPUS_OK);
 	ERR_FAIL_COND(m_decoder == nullptr);
 
@@ -30,79 +32,111 @@ Opus::Opus()
 	err = opus_encoder_ctl(m_encoder, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE));
 	ERR_FAIL_COND(err < 0);
 
-	err = opus_encoder_ctl(m_encoder, OPUS_SET_BITRATE(24000));
+	err = opus_encoder_ctl(m_encoder, OPUS_SET_BITRATE(28000));
 	ERR_FAIL_COND(err < 0);
 
-	m_encodeInputBuffer.resize(SampleFrames);
-	m_encodeOutputBuffer.resize(SampleFrames);
-	m_decodeOutputBuffer.resize(SampleFrames);
+	// Speex
+	m_encodeResampler = speex_resampler_init(2, m_inputMixRate, 48000, 10, &err);
+	ERR_FAIL_COND(err != 0);
+
+	m_decodeResampler = speex_resampler_init(2, 48000, m_outputMixRate, 10, &err);
+	ERR_FAIL_COND(err != 0);
+
+	// Setup buffers
+	m_encodeSampleBuffer.resize(SampleFrames);
+	m_decodeSampleBuffer.resize(SampleFrames);
 }
 
 Opus::~Opus()
 {
 	opus_encoder_destroy(m_encoder);
 	opus_decoder_destroy(m_decoder);
+	speex_resampler_destroy(m_encodeResampler);
+	speex_resampler_destroy(m_decodeResampler);
 }
 
-PackedFloat32Array Opus::encode(PackedVector2Array input)
+void Opus::update_mix_rate(size_t input, size_t output)
 {
-	if (input.size() < SampleFrames) {
-		return {};
-	}
+	int err{};
+	m_inputMixRate = input;
+	m_outputMixRate = output;
 
-	for (size_t i = 0; i < SampleFrames; i++) {
-		m_encodeInputBuffer[i] = input[i].x;
-	}
+	speex_resampler_destroy(m_encodeResampler);
+	m_encodeResampler = speex_resampler_init(2, m_inputMixRate, 48000, 10, &err);
+	ERR_FAIL_COND(err != 0);
 
-	const auto r = opus_encode_float(
-		m_encoder,
-		m_encodeInputBuffer.data(),
-		SampleFrames,
-		m_encodeOutputBuffer.data(),
-		m_encodeOutputBuffer.size()
+	speex_resampler_destroy(m_decodeResampler);
+	m_decodeResampler = speex_resampler_init(2, 48000, m_outputMixRate, 10, &err);
+	ERR_FAIL_COND(err != 0);
+
+	/*
+	UtilityFunctions::print(
+		(std::string("Input encoder mix rate set to: ") + std::to_string(m_inputMixRate)).c_str()
 	);
-	if (r == -1) {
-		return {};
-	}
-
-	auto outputArray = PackedFloat32Array{};
-	outputArray.resize(r);
-	for (size_t i = 0; i < r; i++) {
-		outputArray[i] = m_encodeOutputBuffer[i];
-	}
-
-	return outputArray;
+	UtilityFunctions::print(
+		(std::string("Output encoder mix rate set to: ") + std::to_string(m_outputMixRate)).c_str()
+	);
+	*/
 }
 
-PackedVector2Array Opus::decode(PackedFloat32Array input)
+PackedByteArray Opus::encode(PackedVector2Array samples)
 {
-	std::vector<unsigned char> inputData(input.size());
-	for (size_t i = 0; i < input.size(); i++) {
-		inputData[i] = input[i];
-	}
+	PackedByteArray encoded;
+	encoded.resize(sizeof(float) * SampleFrames * 2);
 
-	const auto r = opus_decode_float(
+	unsigned int inlen = samples.size();
+	unsigned int outlen = SampleFrames;
+
+	speex_resampler_process_interleaved_float(
+		m_encodeResampler,
+		(float*) samples.ptr(),
+		&inlen,
+		(float*) m_encodeSampleBuffer.ptrw(),
+		&outlen
+	);
+
+	const auto encodedSize = opus_encode_float(
+		m_encoder,
+		(float*) m_encodeSampleBuffer.ptr(),
+		SampleFrames,
+		(unsigned char*) encoded.ptrw(),
+		encoded.size()
+	);
+	encoded.resize(encodedSize);
+
+	return encoded;
+}
+
+PackedVector2Array Opus::decode(PackedByteArray encoded)
+{
+	PackedVector2Array output;
+	output.resize(SampleFrames * m_outputMixRate / 48000);
+
+	opus_decode_float(
 		m_decoder,
-		inputData.data(),
-		input.size(),
-		m_decodeOutputBuffer.data(),
+		encoded.ptr(),
+		encoded.size(),
+		(float*) m_decodeSampleBuffer.ptrw(),
 		SampleFrames,
 		0
 	);
-	if (r != SampleFrames) {
-		return {};
-	}
 
-	auto packedOutput = PackedVector2Array{};
-	packedOutput.resize(r);
-	for (size_t i = 0; i < r; i++) {
-		packedOutput[i] = Vector2{m_decodeOutputBuffer[i], m_decodeOutputBuffer[i]};
-	}
+	unsigned int inlen = m_decodeSampleBuffer.size();
+	unsigned int outlen = output.size();
 
-	return packedOutput;
+	speex_resampler_process_interleaved_float(
+		m_decodeResampler,
+		(float*) m_decodeSampleBuffer.ptr(),
+		&inlen,
+		(float*) output.ptrw(),
+		&outlen
+	);
+	output.resize(outlen);
+
+	return output;
 }
 
-void Opus::decode_and_play(Ref<AudioStreamGeneratorPlayback> buffer, PackedFloat32Array input)
+void Opus::decode_and_play(Ref<AudioStreamGeneratorPlayback> buffer, PackedByteArray input)
 {
 	const auto decoded = decode(input);
 	buffer->push_buffer(decoded);

src/GodotOpus.h

@@ -6,6 +6,7 @@
 #include <godot_cpp/classes/audio_stream_generator_playback.hpp>
 
 #include "opus.h"
+#include "speex/speex_resampler.h"
 
 namespace godot {
 
@@ -22,19 +23,25 @@ public:
 	Opus();
 	~Opus();
 
-	PackedFloat32Array encode(PackedVector2Array input);
-	PackedVector2Array decode(PackedFloat32Array input);
+	void update_mix_rate(size_t input, size_t output);
 
-	void decode_and_play(Ref<AudioStreamGeneratorPlayback> buffer, PackedFloat32Array input);
+	PackedByteArray encode(PackedVector2Array input);
+	PackedVector2Array decode(PackedByteArray input);
+
+	void decode_and_play(Ref<AudioStreamGeneratorPlayback> buffer, PackedByteArray input);
 
 private:
-	std::vector<float> m_encodeInputBuffer;
-	std::vector<float> m_decodeOutputBuffer;
-	std::vector<unsigned char> m_encodeOutputBuffer;
+	PackedVector2Array m_encodeSampleBuffer;
+	PackedVector2Array m_decodeSampleBuffer;
+
+	size_t m_outputMixRate{44100};
+	size_t m_inputMixRate{44100};
 
 	OpusEncoder* m_encoder;
 	OpusDecoder* m_decoder;
 
+	SpeexResamplerState* m_encodeResampler;
+	SpeexResamplerState* m_decodeResampler;
 };
 
 }

src/register_types.cpp

@@ -12,6 +12,7 @@
 #include <godot_cpp/variant/utility_functions.hpp>
 
 #include "GodotOpus.h"
+#include "AudioProcessor.h"
 
 using namespace godot;
 
@@ -22,6 +23,7 @@ void gdextension_initialize(ModuleInitializationLevel p_level)
 	if (p_level == MODULE_INITIALIZATION_LEVEL_SCENE)
 	{
 		ClassDB::register_class<Opus>();
+		ClassDB::register_class<AudioProcessor>();
 	}
 }