Re-implement audio capturing based on the AudioService API (fixes issue #2755)

libcef/browser/audio_capturer.cc

@@ -0,0 +1,125 @@
+// Copyright (c) 2019 The Chromium Embedded Framework Authors.
+// Portions copyright (c) 2011 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "libcef/browser/audio_capturer.h"
+#include "libcef/browser/browser_host_impl.h"
+
+#include "components/mirroring/service/captured_audio_input.h"
+#include "content/public/browser/audio_loopback_stream_creator.h"
+#include "media/audio/audio_input_device.h"
+
+namespace {
+
+media::ChannelLayout TranslateChannelLayout(
+    cef_channel_layout_t channel_layout) {
+  // Verify that our enum matches Chromium's values. The enum values match
+  // between those enums and existing values don't ever change, so it's enough
+  // to check that there are no new ones added.
+  static_assert(
+      static_cast<int>(CEF_CHANNEL_LAYOUT_MAX) ==
+          static_cast<int>(media::CHANNEL_LAYOUT_MAX),
+      "cef_channel_layout_t must match the ChannelLayout enum in Chromium");
+  return static_cast<media::ChannelLayout>(channel_layout);
+}
+
+void StreamCreatorHelper(
+    content::WebContents* source_web_contents,
+    content::AudioLoopbackStreamCreator* audio_stream_creator,
+    mojo::PendingRemote<mirroring::mojom::AudioStreamCreatorClient> client,
+    const media::AudioParameters& params,
+    uint32_t total_segments) {
+  audio_stream_creator->CreateLoopbackStream(
+      source_web_contents, params, total_segments,
+      base::BindRepeating(
+          [](mojo::PendingRemote<mirroring::mojom::AudioStreamCreatorClient>
+                 client,
+             mojo::PendingRemote<media::mojom::AudioInputStream> stream,
+             mojo::PendingReceiver<media::mojom::AudioInputStreamClient>
+                 client_receiver,
+             media::mojom::ReadOnlyAudioDataPipePtr data_pipe) {
+            mojo::Remote<mirroring::mojom::AudioStreamCreatorClient>
+                audio_client(std::move(client));
+            audio_client->StreamCreated(
+                std::move(stream), std::move(client_receiver),
+                std::move(data_pipe), false /* initially_muted */);
+          },
+          base::Passed(&client)));
+}
+
+}  // namespace
+
+CefAudioCapturer::CefAudioCapturer(const CefAudioParameters& params,
+                                   CefRefPtr<CefBrowserHostImpl> browser,
+                                   CefRefPtr<CefAudioHandler> audio_handler)
+    : params_(params),
+      browser_(browser),
+      audio_handler_(audio_handler),
+      audio_stream_creator_(content::AudioLoopbackStreamCreator::
+                                CreateInProcessAudioLoopbackStreamCreator()) {
+  media::AudioParameters audio_params(
+      media::AudioParameters::AUDIO_PCM_LINEAR,
+      TranslateChannelLayout(params.channel_layout), params.sample_rate,
+      params.frames_per_buffer);
+
+  if (!audio_params.IsValid()) {
+    LOG(ERROR) << "Invalid audio parameters";
+    return;
+  }
+
+  DCHECK(browser_);
+  DCHECK(audio_handler_);
+  DCHECK(browser_->web_contents());
+
+  channels_ = audio_params.channels();
+  audio_input_device_ = new media::AudioInputDevice(
+      std::make_unique<mirroring::CapturedAudioInput>(base::BindRepeating(
+          &StreamCreatorHelper, base::Unretained(browser_->web_contents()),
+          base::Unretained(audio_stream_creator_.get()))),
+      media::AudioInputDevice::kLoopback);
+
+  audio_input_device_->Initialize(audio_params, this);
+  audio_input_device_->Start();
+}
+
+CefAudioCapturer::~CefAudioCapturer() {
+  StopStream();
+}
+
+void CefAudioCapturer::OnCaptureStarted() {
+  audio_handler_->OnAudioStreamStarted(browser_, params_, channels_);
+  DCHECK(!capturing_);
+  capturing_ = true;
+}
+
+void CefAudioCapturer::Capture(const media::AudioBus* source,
+                               base::TimeTicks audio_capture_time,
+                               double /*volume*/,
+                               bool /*key_pressed*/) {
+  const int channels = source->channels();
+  std::array<const float*, media::CHANNELS_MAX> data;
+  DCHECK(channels == channels_);
+  DCHECK(channels <= static_cast<int>(data.size()));
+  for (int c = 0; c < channels; ++c) {
+    data[c] = source->channel(c);
+  }
+  base::TimeDelta pts = audio_capture_time - base::TimeTicks::UnixEpoch();
+  audio_handler_->OnAudioStreamPacket(browser_, data.data(), source->frames(),
+                                      pts.InMilliseconds());
+}
+
+void CefAudioCapturer::OnCaptureError(const std::string& message) {
+  audio_handler_->OnAudioStreamError(browser_, message);
+  StopStream();
+}
+
+void CefAudioCapturer::StopStream() {
+  if (audio_input_device_)
+    audio_input_device_->Stop();
+  if (capturing_)
+    audio_handler_->OnAudioStreamStopped(browser_);
+
+  audio_input_device_ = nullptr;
+  capturing_ = false;
+}

libcef/browser/audio_capturer.h

@@ -0,0 +1,53 @@
+// Copyright (c) 2019 The Chromium Embedded Framework Authors.
+// Portions copyright (c) 2011 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CEF_LIBCEF_BROWSER_AUDIO_CAPTURER_H_
+#define CEF_LIBCEF_BROWSER_AUDIO_CAPTURER_H_
+#pragma once
+
+#include "include/internal/cef_ptr.h"
+#include "include/internal/cef_types_wrappers.h"
+
+#include "media/base/audio_capturer_source.h"
+
+namespace content {
+class AudioLoopbackStreamCreator;
+}  // namespace content
+
+namespace media {
+class AudioInputDevice;
+}  // namespace media
+
+class CefAudioHandler;
+class CefBrowserHostImpl;
+
+class CefAudioCapturer : public media::AudioCapturerSource::CaptureCallback {
+ public:
+  CefAudioCapturer(const CefAudioParameters& params,
+                   CefRefPtr<CefBrowserHostImpl> browser,
+                   CefRefPtr<CefAudioHandler> audio_handler);
+  ~CefAudioCapturer() override;
+
+ private:
+  void OnCaptureStarted() override;
+  void Capture(const media::AudioBus* audio_source,
+               base::TimeTicks audio_capture_time,
+               double volume,
+               bool key_pressed) override;
+  void OnCaptureError(const std::string& message) override;
+  void OnCaptureMuted(bool is_muted) override {}
+
+  void StopStream();
+
+  CefAudioParameters params_;
+  CefRefPtr<CefBrowserHostImpl> browser_;
+  CefRefPtr<CefAudioHandler> audio_handler_;
+  std::unique_ptr<content::AudioLoopbackStreamCreator> audio_stream_creator_;
+  scoped_refptr<media::AudioInputDevice> audio_input_device_;
+  bool capturing_ = false;
+  int channels_ = 0;
+};
+
+#endif  // CEF_LIBCEF_BROWSER_AUDIO_CAPTURER_H_

libcef/browser/browser_host_impl.cc

@@ -8,6 +8,7 @@
 #include <string>
 #include <utility>
 
+#include "libcef/browser/audio_capturer.h"
 #include "libcef/browser/browser_context.h"
 #include "libcef/browser/browser_info.h"
 #include "libcef/browser/browser_info_manager.h"
@@ -194,6 +195,9 @@ void OnDownloadImage(uint32 max_image_size,
                                     image_impl.get());
 }
 
+static constexpr base::TimeDelta kRecentlyAudibleTimeout =
+    base::TimeDelta::FromSeconds(2);
+
 }  // namespace
 
 // CefBrowserHost static methods.
@@ -1559,6 +1563,10 @@ void CefBrowserHostImpl::DestroyBrowser() {
   javascript_dialog_manager_.reset(nullptr);
   menu_manager_.reset(nullptr);
 
+  // Delete the audio capturer
+  recently_audible_timer_.Stop();
+  audio_capturer_.reset(nullptr);
+
   // Delete the platform delegate.
   platform_delegate_.reset(nullptr);
 
@@ -2697,6 +2705,25 @@ void CefBrowserHostImpl::DidUpdateFaviconURL(
   }
 }
 
+void CefBrowserHostImpl::OnAudioStateChanged(bool audible) {
+  if (audible) {
+    recently_audible_timer_.Stop();
+    StartAudioCapturer();
+  } else if (audio_capturer_) {
+    // If you have a media playing that has a short quiet moment, web_contents
+    // will immediately switch to non-audible state. We don't want to stop
+    // audio stream so quickly, let's give the stream some time to resume
+    // playing.
+    recently_audible_timer_.Start(
+        FROM_HERE, kRecentlyAudibleTimeout,
+        base::BindOnce(&CefBrowserHostImpl::OnRecentlyAudibleTimerFired, this));
+  }
+}
+
+void CefBrowserHostImpl::OnRecentlyAudibleTimerFired() {
+  audio_capturer_.reset();
+}
+
 bool CefBrowserHostImpl::OnMessageReceived(const IPC::Message& message) {
   // Handle the cursor message here if mouse cursor change is disabled instead
   // of propegating the message to the normal handler.
@@ -2792,6 +2819,25 @@ bool CefBrowserHostImpl::HasObserver(Observer* observer) const {
   return observers_.HasObserver(observer);
 }
 
+void CefBrowserHostImpl::StartAudioCapturer() {
+  if (!client_.get() || audio_capturer_)
+    return;
+
+  CefRefPtr<CefAudioHandler> audio_handler = client_->GetAudioHandler();
+  if (!audio_handler.get())
+    return;
+
+  CefAudioParameters params;
+  params.channel_layout = CEF_CHANNEL_LAYOUT_STEREO;
+  params.sample_rate = media::AudioParameters::kAudioCDSampleRate;
+  params.frames_per_buffer = 1024;
+
+  if (!audio_handler->GetAudioParameters(this, params))
+    return;
+
+  audio_capturer_.reset(new CefAudioCapturer(params, this, audio_handler));
+}
+
 CefBrowserHostImpl::NavigationLock::NavigationLock(
     CefRefPtr<CefBrowserHostImpl> browser)
     : browser_(browser) {

libcef/browser/browser_host_impl.h

@@ -48,6 +48,7 @@ class Widget;
 }
 #endif  // defined(USE_AURA)
 
+class CefAudioCapturer;
 class CefBrowserInfo;
 class CefBrowserPlatformDelegate;
 class CefDevToolsFrontend;
@@ -486,6 +487,7 @@ class CefBrowserHostImpl : public CefBrowserHost,
                      base::ProcessId plugin_pid) override;
   void DidUpdateFaviconURL(
       const std::vector<blink::mojom::FaviconURLPtr>& candidates) override;
+  void OnAudioStateChanged(bool audible) override;
   bool OnMessageReceived(const IPC::Message& message) override;
   bool OnMessageReceived(const IPC::Message& message,
                          content::RenderFrameHost* render_frame_host) override;
@@ -505,6 +507,7 @@ class CefBrowserHostImpl : public CefBrowserHost,
   void AddObserver(Observer* observer);
   void RemoveObserver(Observer* observer);
   bool HasObserver(Observer* observer) const;
+
   class NavigationLock final {
    private:
     friend class CefBrowserHostImpl;
@@ -585,6 +588,10 @@ class CefBrowserHostImpl : public CefBrowserHost,
 
   void ConfigureAutoResize();
 
+  void StartAudioCapturer();
+
+  void OnRecentlyAudibleTimerFired();
+
   CefBrowserSettings settings_;
   CefRefPtr<CefClient> client_;
   scoped_refptr<CefBrowserInfo> browser_info_;
@@ -667,6 +674,14 @@ class CefBrowserHostImpl : public CefBrowserHost,
   CefRefPtr<CefExtension> extension_;
   bool is_background_host_ = false;
 
+  // Used for capturing audio for CefAudioHandler.
+  std::unique_ptr<CefAudioCapturer> audio_capturer_;
+
+  // Timer for determining when "recently audible" transitions to false. This
+  // starts running when a tab stops being audible, and is canceled if it starts
+  // being audible again before it fires.
+  base::OneShotTimer recently_audible_timer_;
+
   // Used with auto-resize.
   bool auto_resize_enabled_ = false;
   gfx::Size auto_resize_min_;