From 189b247282f38f0b4cdfd77c7f31aab2ea71d6cb Mon Sep 17 00:00:00 2001 From: reito Date: Tue, 3 Dec 2024 12:55:24 -0500 Subject: [PATCH] osr: Add more info in accelerated callback (fixes #3730) --- cef_paths2.gypi | 3 +- include/cef_api_hash.h | 8 +- include/internal/cef_types_linux.h | 6 ++ include/internal/cef_types_mac.h | 6 ++ include/internal/cef_types_osr.h | 105 +++++++++++++++++++++++ include/internal/cef_types_win.h | 6 ++ libcef/browser/osr/video_consumer_osr.cc | 92 ++++++++++++++------ 7 files changed, 194 insertions(+), 32 deletions(-) create mode 100644 include/internal/cef_types_osr.h diff --git a/cef_paths2.gypi b/cef_paths2.gypi index c38f13d8c..88fc7b230 100644 --- a/cef_paths2.gypi +++ b/cef_paths2.gypi @@ -55,10 +55,11 @@ 'include/internal/cef_time.h', 'include/internal/cef_trace_event_internal.h', 'include/internal/cef_types.h', + 'include/internal/cef_types_color.h', 'include/internal/cef_types_content_settings.h', 'include/internal/cef_types_geometry.h', + 'include/internal/cef_types_osr.h', 'include/internal/cef_types_runtime.h', - 'include/internal/cef_types_color.h', ], 'includes_capi': [ 'include/capi/cef_base_capi.h', diff --git a/include/cef_api_hash.h b/include/cef_api_hash.h index b1c434f6a..7867f7a1a 100644 --- a/include/cef_api_hash.h +++ b/include/cef_api_hash.h @@ -42,13 +42,13 @@ // way that may cause binary incompatibility with other builds. The universal // hash value will change if any platform is affected whereas the platform hash // values will change only if that particular platform is affected. -#define CEF_API_HASH_UNIVERSAL "7ce563b3df3a227c06d331774d5fa51c8421a7f1" +#define CEF_API_HASH_UNIVERSAL "3e3393f10c4b95f7516521c8643b9a735fd0c3f3" #if defined(OS_WIN) -#define CEF_API_HASH_PLATFORM "0b070263fe1c7304bd90bd7a8338fcc08410d299" +#define CEF_API_HASH_PLATFORM "aafe004dac5cf8b7f0b5ef12518c4a16e150f510" #elif defined(OS_MAC) -#define CEF_API_HASH_PLATFORM "e99938b49f333ccda55ea34dfb36dd65a6b2cacd" +#define CEF_API_HASH_PLATFORM "9cd794a0ab4506060ca2d7b3c335778d026337a4" #elif defined(OS_LINUX) -#define CEF_API_HASH_PLATFORM "45669721b97760d8cda9ea29a5bd070d38267424" +#define CEF_API_HASH_PLATFORM "fab73fe3fddb82c0f75a404fb464935592880803" #endif #ifdef __cplusplus diff --git a/include/internal/cef_types_linux.h b/include/internal/cef_types_linux.h index 84972cd7f..b9bd64b30 100644 --- a/include/internal/cef_types_linux.h +++ b/include/internal/cef_types_linux.h @@ -44,6 +44,7 @@ typedef struct _XDisplay XDisplay; #include "include/internal/cef_string.h" #include "include/internal/cef_types_color.h" #include "include/internal/cef_types_geometry.h" +#include "include/internal/cef_types_osr.h" #include "include/internal/cef_types_runtime.h" // Handle types. @@ -193,6 +194,11 @@ typedef struct _cef_accelerated_paint_info_t { /// The pixel format of the texture. /// cef_color_type_t format; + + /// + /// The extra common info. + /// + cef_accelerated_paint_info_common_t extra; } cef_accelerated_paint_info_t; #ifdef __cplusplus diff --git a/include/internal/cef_types_mac.h b/include/internal/cef_types_mac.h index 0f5343d81..0e9c88014 100644 --- a/include/internal/cef_types_mac.h +++ b/include/internal/cef_types_mac.h @@ -37,6 +37,7 @@ #include "include/internal/cef_string.h" #include "include/internal/cef_types_color.h" #include "include/internal/cef_types_geometry.h" +#include "include/internal/cef_types_osr.h" #include "include/internal/cef_types_runtime.h" // Handle types. @@ -162,6 +163,11 @@ typedef struct _cef_accelerated_paint_info_t { /// The pixel format of the texture. /// cef_color_type_t format; + + /// + /// The extra common info. + /// + cef_accelerated_paint_info_common_t extra; } cef_accelerated_paint_info_t; #ifdef __cplusplus diff --git a/include/internal/cef_types_osr.h b/include/internal/cef_types_osr.h new file mode 100644 index 000000000..35d1a0d1d --- /dev/null +++ b/include/internal/cef_types_osr.h @@ -0,0 +1,105 @@ +// Copyright (c) 2024 Marshall A. Greenblatt. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the name Chromium Embedded +// Framework nor the names of its contributors may be used to endorse +// or promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef CEF_INCLUDE_INTERNAL_CEF_TYPES_OSR_H_ +#define CEF_INCLUDE_INTERNAL_CEF_TYPES_OSR_H_ +#pragma once + +#include "include/internal/cef_types_geometry.h" + +/// +/// Structure containing shared texture common metadata. +/// For documentation on each field, please refer to +/// src/media/base/video_frame_metadata.h for actual details. +/// +typedef struct _cef_accelerated_paint_info_common_t { + /// + /// Timestamp of the frame in microseconds since capture start. + /// + uint64_t timestamp; + + /// + /// The full dimensions of the video frame. + /// + cef_size_t coded_size; + + /// + /// The visible area of the video frame. + /// + cef_rect_t visible_rect; + + /// + /// The region of the video frame that capturer would like to populate. + /// + cef_rect_t content_rect; + + /// + /// Full size of the source frame. + /// + cef_size_t source_size; + + /// + /// Updated area of frame, can be considered as the `dirty` area. + /// + cef_rect_t capture_update_rect; + + /// + /// May reflects where the frame's contents originate from if region + /// capture is used internally. + /// + cef_rect_t region_capture_rect; + + /// + /// The increamental counter of the frame. + /// + uint64_t capture_counter; + + /// + /// Optional flag of capture_update_rect + /// + uint8_t has_capture_update_rect; + + /// + /// Optional flag of region_capture_rect + /// + uint8_t has_region_capture_rect; + + /// + /// Optional flag of source_size + /// + uint8_t has_source_size; + + /// + /// Optional flag of capture_counter + /// + uint8_t has_capture_counter; + +} cef_accelerated_paint_info_common_t; + +#endif // CEF_INCLUDE_INTERNAL_CEF_TYPES_OSR_H_ diff --git a/include/internal/cef_types_win.h b/include/internal/cef_types_win.h index eeb3a0690..13e6ed8be 100644 --- a/include/internal/cef_types_win.h +++ b/include/internal/cef_types_win.h @@ -39,6 +39,7 @@ #include "include/internal/cef_string.h" #include "include/internal/cef_types_color.h" #include "include/internal/cef_types_geometry.h" +#include "include/internal/cef_types_osr.h" #include "include/internal/cef_types_runtime.h" // Handle types. @@ -130,6 +131,11 @@ typedef struct _cef_accelerated_paint_info_t { /// The pixel format of the texture. /// cef_color_type_t format; + + /// + /// The extra common info. + /// + cef_accelerated_paint_info_common_t extra; } cef_accelerated_paint_info_t; #ifdef __cplusplus diff --git a/libcef/browser/osr/video_consumer_osr.cc b/libcef/browser/osr/video_consumer_osr.cc index 64c4d90a7..de8e085b8 100644 --- a/libcef/browser/osr/video_consumer_osr.cc +++ b/libcef/browser/osr/video_consumer_osr.cc @@ -99,6 +99,30 @@ void CefVideoConsumerOSR::OnFrameCaptured( callbacks) { ScopedVideoFrameDone scoped_done(std::move(callbacks)); + media::VideoFrameMetadata metadata = info->metadata; + gfx::Rect damage_rect; + + if (bounds_in_pixels_) { + // Use the bounds passed to RequestRefreshFrame(). + damage_rect = gfx::Rect(info->coded_size); + damage_rect.Intersect(*bounds_in_pixels_); + bounds_in_pixels_ = std::nullopt; + } else { + // Retrieve the rectangular region of the frame that has changed since the + // frame with the directly preceding CAPTURE_COUNTER. If that frame was not + // received, typically because it was dropped during transport from the + // producer, clients must assume that the entire frame has changed. + // This rectangle is relative to the full frame data, i.e. [0, 0, + // coded_size.width(), coded_size.height()]. It does not have to be + // fully contained within visible_rect. + if (metadata.capture_update_rect) { + damage_rect = *metadata.capture_update_rect; + } + if (damage_rect.IsEmpty()) { + damage_rect = gfx::Rect(info->coded_size); + } + } + // If it is GPU Texture OSR. if (use_shared_texture_) { CHECK(data->is_gpu_memory_buffer_handle() && @@ -112,24 +136,61 @@ void CefVideoConsumerOSR::OnFrameCaptured( ? CEF_COLOR_TYPE_RGBA_8888 : CEF_COLOR_TYPE_BGRA_8888; + // Build extra common info. + cef_accelerated_paint_info_common_t extra = {}; + extra.timestamp = info->timestamp.InMicroseconds(); + extra.coded_size = {info->coded_size.width(), info->coded_size.height()}; + extra.visible_rect = {info->visible_rect.x(), info->visible_rect.y(), + info->visible_rect.width(), + info->visible_rect.height()}; + extra.content_rect = {content_rect.x(), content_rect.y(), + content_rect.width(), content_rect.height()}; + extra.timestamp = info->timestamp.InMicroseconds(); + + extra.has_capture_counter = info->metadata.capture_counter.has_value(); + extra.has_capture_update_rect = + info->metadata.capture_update_rect.has_value(); + extra.has_region_capture_rect = + info->metadata.region_capture_rect.has_value(); + extra.has_source_size = info->metadata.source_size.has_value(); + + extra.capture_counter = info->metadata.capture_counter.value_or(0); + if (extra.has_capture_update_rect) { + auto rect = info->metadata.capture_update_rect.value(); + extra.capture_update_rect = {rect.x(), rect.y(), rect.width(), + rect.height()}; + } + if (extra.has_region_capture_rect) { + auto rect = info->metadata.region_capture_rect.value(); + extra.region_capture_rect = {rect.x(), rect.y(), rect.width(), + rect.height()}; + } + if (extra.has_source_size) { + auto size = info->metadata.source_size.value(); + extra.source_size = {size.width(), size.height()}; + } + #if BUILDFLAG(IS_WIN) auto& gmb_handle = data->get_gpu_memory_buffer_handle(); cef_accelerated_paint_info_t paint_info; + paint_info.extra = extra; paint_info.shared_texture_handle = gmb_handle.dxgi_handle.Get(); paint_info.format = pixel_format; - view_->OnAcceleratedPaint(content_rect, info->coded_size, paint_info); + view_->OnAcceleratedPaint(damage_rect, info->coded_size, paint_info); #elif BUILDFLAG(IS_APPLE) auto& gmb_handle = data->get_gpu_memory_buffer_handle(); cef_accelerated_paint_info_t paint_info; + paint_info.extra = extra; paint_info.shared_texture_io_surface = gmb_handle.io_surface.get(); paint_info.format = pixel_format; - view_->OnAcceleratedPaint(content_rect, info->coded_size, paint_info); + view_->OnAcceleratedPaint(damage_rect, info->coded_size, paint_info); #elif BUILDFLAG(IS_LINUX) auto& gmb_handle = data->get_gpu_memory_buffer_handle(); auto& native_pixmap = gmb_handle.native_pixmap_handle; CHECK(native_pixmap.planes.size() <= kAcceleratedPaintMaxPlanes); cef_accelerated_paint_info_t paint_info; + paint_info.extra = extra; paint_info.plane_count = native_pixmap.planes.size(); paint_info.modifier = native_pixmap.modifier; paint_info.format = pixel_format; @@ -143,11 +204,12 @@ void CefVideoConsumerOSR::OnFrameCaptured( cef_plane.fd = plane.fd.get(); paint_info.planes[cef_plain_index++] = cef_plane; } - view_->OnAcceleratedPaint(content_rect, info->coded_size, paint_info); + view_->OnAcceleratedPaint(damage_rect, info->coded_size, paint_info); #endif return; } + // If it is CPU bitmap OSR. if (info->pixel_format != media::PIXEL_FORMAT_ARGB) { DLOG(ERROR) << "Unsupported pixel format " << info->pixel_format; return; @@ -179,29 +241,5 @@ void CefVideoConsumerOSR::OnFrameCaptured( // API requires a non-const pointer. So, cast away the const. void* const pixels = const_cast(mapping.memory()); - media::VideoFrameMetadata metadata = info->metadata; - gfx::Rect damage_rect; - - if (bounds_in_pixels_) { - // Use the bounds passed to RequestRefreshFrame(). - damage_rect = gfx::Rect(info->coded_size); - damage_rect.Intersect(*bounds_in_pixels_); - bounds_in_pixels_ = std::nullopt; - } else { - // Retrieve the rectangular region of the frame that has changed since the - // frame with the directly preceding CAPTURE_COUNTER. If that frame was not - // received, typically because it was dropped during transport from the - // producer, clients must assume that the entire frame has changed. - // This rectangle is relative to the full frame data, i.e. [0, 0, - // coded_size.width(), coded_size.height()]. It does not have to be - // fully contained within visible_rect. - if (metadata.capture_update_rect) { - damage_rect = *metadata.capture_update_rect; - } - if (damage_rect.IsEmpty()) { - damage_rect = gfx::Rect(info->coded_size); - } - } - view_->OnPaint(damage_rect, info->coded_size, pixels); }