Introduce extremely simplistic audio capturing mechanism

SConstruct

@@ -18,7 +18,9 @@ includes += Glob("src/platform/win32/*.h")
 # Append additional library paths and libraries for Opus, Speex, and vpx
 env.Append(CPPPATH=['#3rdparty/opus/include', '#3rdparty/speex/include', '#3rdparty/libvpx/include'])
 env.Append(LIBPATH=['#3rdparty/opus/lib', '#3rdparty/speex/lib', '#3rdparty/libvpx/lib/x64'])
-env.Append(LIBS=['opus', 'libspeex', 'libspeexdsp', 'vpx'])
+env.Append(LIBS=['opus', 'libspeex', 'libspeexdsp', 'libvpx'])
+
+env.Append(CCFLAGS='/MD')
 
 # Determine extension and addon path
 (extension_path,) = glob("export/addons/*/*.gdextension")

godot-cpp

@@ -1 +1 @@
-Subproject commit 78ffea5b136f3178c31cddb28f6b963ceaa89420
+Subproject commit 9da6ecd14485a3dacc04d8e0558c21beb709ca9f

src/AudioCapture.cpp

@@ -0,0 +1,149 @@
+#include "AudioCapture.h"
+
+#include <windows.h>
+#include <mmdeviceapi.h>
+#include <audioclient.h>
+#include <iostream>
+
+#include <godot_cpp/variant/utility_functions.hpp>
+
+namespace godot {
+
+
+void AudioCapture::_bind_methods()
+{
+  ClassDB::bind_method(D_METHOD("process"), &AudioCapture::process);
+  ClassDB::bind_method(D_METHOD("start"), &AudioCapture::start);
+  ClassDB::bind_method(D_METHOD("get_frame"), &AudioCapture::getFrame);
+}
+
+AudioCapture::AudioCapture()
+{
+  UtilityFunctions::print("BEDZIE RUCHANES");
+
+  HRESULT hr;
+  IMMDeviceEnumerator* pEnumerator = nullptr;
+  IMMDevice* pDevice = nullptr;
+
+  hr = CoInitialize(nullptr);
+  if (FAILED(hr)) {
+    UtilityFunctions::printerr("CoInitialize failed:", hr);
+    return;
+  }
+
+  hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, IID_PPV_ARGS(&pEnumerator));
+  if (FAILED(hr)) {
+    UtilityFunctions::printerr("CoCreateInstance failed:", hr);
+    return;
+  }
+
+  hr = pEnumerator->GetDefaultAudioEndpoint(eRender, eConsole, &pDevice);
+  if (FAILED(hr)) {
+    UtilityFunctions::printerr("GetDefaultAudioEndpoint failed:", hr);
+    return;
+  }
+
+  hr = pDevice->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, (void**)&m_audioClient);
+  if (FAILED(hr)) {
+    UtilityFunctions::printerr("Device activate failed:", hr);
+    return;
+  }
+
+  WAVEFORMATEX* pwfx{nullptr};
+  hr = m_audioClient->GetMixFormat(&pwfx);
+  if (FAILED(hr)) {
+    UtilityFunctions::printerr("GetMixFormat failed:", hr);
+    return;
+  }
+
+  hr = m_audioClient->Initialize(AUDCLNT_SHAREMODE_SHARED, AUDCLNT_STREAMFLAGS_LOOPBACK, 10000000, 0, pwfx, nullptr);
+  if (FAILED(hr)) {
+    UtilityFunctions::printerr("Device initialize failed:", hr);
+    return;
+  };
+
+  hr = m_audioClient->GetService(IID_PPV_ARGS(&m_captureClient));
+  if (FAILED(hr)) {
+    UtilityFunctions::printerr("Get service failed:", hr);
+    return;
+  }
+
+  UtilityFunctions::print("Gites majonez");
+}
+
+Array AudioCapture::getFrame() const
+{
+  // TODO: common pre-resized buffer?
+  Array out{};
+
+  UINT32 len = 0;
+  UINT32 frames = 0;
+  DWORD flags = 0;
+
+  m_captureClient->GetNextPacketSize(&len);
+
+  if (len > 0) {
+    BYTE* pData = nullptr;
+    m_captureClient->GetBuffer(&pData, &frames, &flags, NULL, NULL);
+
+    if (pData && frames > 0) {
+      float* pfData = (float*)pData;
+      for (UINT32 i = 0; i < frames; ++i) {
+        float leftChannel = pfData[i * 2];
+        float rightChannel = pfData[i * 2 + 1];
+        out.push_back(Vector2(leftChannel, rightChannel));
+      }
+    }
+    else {
+      //out.push_back(Vector2(0, 0));
+    }
+
+    m_captureClient->ReleaseBuffer(frames);
+    m_captureClient->GetNextPacketSize(&len);
+  }
+
+  return out;
+}
+
+void AudioCapture::process()
+{
+  UINT32 len = 0;
+  UINT32 frames = 0;
+  DWORD flags = 0;
+
+  m_captureClient->GetNextPacketSize(&len);
+
+  while (len > 0) {
+    BYTE* pData = nullptr;
+    m_captureClient->GetBuffer(&pData, &frames, &flags, NULL, NULL);
+
+    if (pData && frames > 0) {
+      float* pfData = (float*)pData;
+      for (UINT32 i = 0; i < frames; ++i) {
+        float leftChannel = pfData[i * 2];
+        float rightChannel = pfData[i * 2 + 1];
+        m_audioPlayback->push_frame(Vector2(leftChannel, rightChannel));
+      }
+    }
+    else {
+      m_audioPlayback->push_frame(Vector2(0, 0));
+    }
+
+    m_captureClient->ReleaseBuffer(frames);
+    m_captureClient->GetNextPacketSize(&len);
+  }
+
+}
+
+void AudioCapture::start(Ref<AudioStreamGeneratorPlayback> playback)
+{
+  m_audioPlayback = playback;
+
+  const auto hr = m_audioClient->Start();
+  if (FAILED(hr)) {
+    UtilityFunctions::printerr("Start failed:", hr);
+    return;
+  };
+}
+
+}

src/AudioCapture.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <windows.h>
+#include <mmdeviceapi.h>
+#include <audioclient.h>
+
+#include <godot_cpp/classes/node.hpp>
+#include <godot_cpp/core/class_db.hpp>
+#include <godot_cpp/classes/audio_stream_generator.hpp>
+#include <godot_cpp/classes/audio_stream_generator_playback.hpp>
+
+namespace godot {
+
+class AudioCapture : public Node
+{
+	GDCLASS(AudioCapture, Node);
+
+protected:
+	static void _bind_methods();
+
+public:
+	AudioCapture();
+
+	Array getFrame() const;
+
+	void process();
+  void start(Ref<AudioStreamGeneratorPlayback> playback);
+
+private:
+	IAudioClient* m_audioClient = nullptr;
+	IAudioCaptureClient* m_captureClient = nullptr;
+
+  Ref<AudioStreamGenerator> m_audioStream;
+  Ref<AudioStreamGeneratorPlayback> m_audioPlayback;
+
+
+};
+
+}

src/GodotObs.cpp

@@ -52,6 +52,7 @@ void Obs::_bind_methods()
 	ClassDB::bind_method(D_METHOD("get_screen_frame"), &Obs::getEncodedScreenFrame);
   ClassDB::bind_method(D_METHOD("render_frame"), &Obs::renderFrameToMesh);
   ClassDB::bind_method(D_METHOD("get_screen_count"), &Obs::getScreenCount);
+  ClassDB::bind_method(D_METHOD("start_capture"), &Obs::startCapture);
 }
 
 
@@ -92,6 +93,12 @@ Obs::~Obs()
 }
 
 
+bool Obs::startCapture()
+{
+  return m_capturer.init();
+}
+
+
 PackedByteArray Obs::getEncodedScreenFrame(size_t id)
 {
 #ifdef PROFILER_ENABLED

src/GodotObs.h

@@ -28,9 +28,10 @@ public:
 	Obs();
 	~Obs();
 
+	bool startCapture();
+
 	PackedByteArray getEncodedScreenFrame(size_t id);
 	void renderFrameToMesh(PackedByteArray frame, Ref<StandardMaterial3D> mat);
-
 	size_t getScreenCount() const;
 
 private:

src/platform/win32/capture_window.cpp

@@ -34,11 +34,7 @@ BOOL CALLBACK MonitorEnumProc(HMONITOR hMonitor, HDC hdcMonitor, LPRECT lprcMoni
 class AcceleratedWindowCapturer
 {
 public:
-  AcceleratedWindowCapturer() {
-    init();
-  }
-
-  void init()
+  bool init()
   {
     D3D_FEATURE_LEVEL featureLevel = D3D_FEATURE_LEVEL_11_0;
     auto hr = D3D11CreateDevice(
@@ -55,21 +51,21 @@ public:
     );
     if (FAILED(hr)) {
       reset();
-      return;
+      return false;
     }
 
     IDXGIDevice* dxgiDevice = nullptr;
     hr = m_device->QueryInterface(__uuidof(IDXGIDevice), reinterpret_cast<void**>(&dxgiDevice));
     if (FAILED(hr)) {
       reset();
-      return;
+      return false;
     }
 
     IDXGIAdapter* dxgiAdapter = nullptr;
     hr = dxgiDevice->GetParent(__uuidof(IDXGIAdapter), reinterpret_cast<void**>(&dxgiAdapter));
     if (FAILED(hr)) {
       reset();
-      return;
+      return false;
     }
     dxgiDevice->Release();
 
@@ -77,7 +73,7 @@ public:
     hr = dxgiAdapter->EnumOutputs(m_currentScreen, &dxgiOutput);
     if (FAILED(hr)) {
       reset();
-      return;
+      return false;
     }
     dxgiAdapter->Release();
 
@@ -85,7 +81,7 @@ public:
     hr = dxgiOutput->QueryInterface(__uuidof(IDXGIOutput1), reinterpret_cast<void**>(&dxgiOutput1));
     if (FAILED(hr)) {
       reset();
-      return;
+      return false;
     }
     dxgiOutput->Release();
 
@@ -93,10 +89,12 @@ public:
     hr = dxgiOutput1->DuplicateOutput(m_device.Get(), &m_duplication);
     if (FAILED(hr)) {
       reset();
-      return;
+      return false;
     }
     dxgiOutput1->Release();
 
+    m_failedAttempts = 0;
+    return true;
   }
 
   Frame nextFrame(size_t outputWidth, size_t outputHeight)
@@ -384,6 +382,10 @@ public:
     m_width = m_height = 0;
     m_frameAcquired = false;
 
+    if (m_failedAttempts++ > 3) {
+      return;
+    }
+
     init();
   }
 
@@ -399,6 +401,8 @@ public:
   }
 
 private:
+  constexpr static auto m_maxFails{ 3 };
+
   Microsoft::WRL::ComPtr<ID3D11Device> m_device;
   Microsoft::WRL::ComPtr<ID3D11DeviceContext> m_context;
   Microsoft::WRL::ComPtr<IDXGIOutputDuplication> m_duplication;
@@ -412,6 +416,8 @@ private:
   size_t m_width{};
   size_t m_height{};
 
+  size_t m_failedAttempts{};
+
   size_t m_currentScreen{};
 };
 
@@ -425,6 +431,11 @@ WindowCapturer::~WindowCapturer()
 {
 }
 
+bool WindowCapturer::init()
+{
+  return m_impl->init();
+}
+
 Frame WindowCapturer::capture(size_t id, size_t width, size_t height)
 {
   if (id != m_impl->getCurrentScreen()) {

src/platform/win32/capture_window.h

@@ -20,6 +20,8 @@ public:
   WindowCapturer();
   ~WindowCapturer();
 
+  bool init();
+
   Frame capture(size_t id, size_t width, size_t height);
 
 private:

src/register_types.cpp

@@ -13,6 +13,7 @@
 
 #include "GodotOpus.h"
 #include "AudioProcessor.h"
+#include "AudioCapture.h"
 #include "GodotObs.h"
 
 using namespace godot;
@@ -25,6 +26,7 @@ void gdextension_initialize(ModuleInitializationLevel p_level)
 	{
 		ClassDB::register_class<Opus>();
 		ClassDB::register_class<AudioProcessor>();
+		ClassDB::register_class<AudioCapture>();
 		ClassDB::register_class<Obs>();
 	}
 }