diff --git a/doc/filters.texi b/doc/filters.texi index 3534c87c..ace8d95a 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -29982,6 +29982,8 @@ window matching the provided regular expressions is captured. This source does NOT hold a stable FPS. It returns frames at whatever rate the compositor provides them, only capped by the @option{max_framerate}. If you need a stable rate, you need to add an fps filter to drop/duplicate frames as needed. +A still source may deliver no frames at all; set @option{dup_frames} so the fps +filter still has frames to duplicate. If the capture source disappears mid-capture (window closed, monitor disconnected), the filter will return EOF. @@ -30056,6 +30058,18 @@ Crop this many pixels from the left side of the captured frames. If set to 1, return frames with premultiplied alpha. Default is 0 (straight alpha). +@item dup_frames +If enabled, output the last frame again whenever the source has not delivered +a new one within one @option{max_framerate} interval, so the output keeps +flowing while the captured content does not change. Until the source delivers +its first frame, black frames are output instead, so the output may begin with +black frames. Unlike the ddagrab option of the same name, this does not make +the output constant rate: new frames still arrive at the compositor's rate. +Repeats follow the system timer, so they average @option{max_framerate} but +are not evenly spaced. Frames may also come less than one interval apart when +a repeat and a new frame coincide; add an @code{fps} filter for a constant +rate. Default is 0. + @item resize_mode Defines how the captured content is fitted into the output canvas size. Possible values: diff --git a/libavfilter/vsrc_gfxcapture.c b/libavfilter/vsrc_gfxcapture.c index 19d8a8a5..10943d06 100644 --- a/libavfilter/vsrc_gfxcapture.c +++ b/libavfilter/vsrc_gfxcapture.c @@ -57,6 +57,8 @@ static const AVOption gfxcapture_options[] = { { "crop_bottom", "number of pixels to crop from the bottom of the captured area", OFFSET(crop_bottom), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS }, { "premultiplied", "return premultiplied alpha frames", OFFSET(premult_alpha), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS }, + { "dup_frames", "repeat the last frame when the source sends none in time", + OFFSET(dup_frames), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS }, { "resize_mode", "capture source resize behavior", OFFSET(resize_mode), AV_OPT_TYPE_INT, { .i64 = GFX_RESIZE_CROP }, 0, GFX_RESIZE_NB - 1, FLAGS, .unit = "resize_mode" }, { "crop", "crop or add black bars into frame", 0, AV_OPT_TYPE_CONST, { .i64 = GFX_RESIZE_CROP }, 0, 0, FLAGS, .unit = "resize_mode" }, { "scale", "scale source to fit initial size", 0, AV_OPT_TYPE_CONST, { .i64 = GFX_RESIZE_SCALE }, 0, 0, FLAGS, .unit = "resize_mode" }, diff --git a/libavfilter/vsrc_gfxcapture.h b/libavfilter/vsrc_gfxcapture.h index f541f9ba..bace2bea 100644 --- a/libavfilter/vsrc_gfxcapture.h +++ b/libavfilter/vsrc_gfxcapture.h @@ -67,6 +67,7 @@ typedef struct GfxCaptureContext { int resize_mode; int scale_mode; int premult_alpha; + int dup_frames; } GfxCaptureContext; #ifdef __cplusplus diff --git a/libavfilter/vsrc_gfxcapture_winrt.cpp b/libavfilter/vsrc_gfxcapture_winrt.cpp index ddc6a7df..e1161ffd 100644 --- a/libavfilter/vsrc_gfxcapture_winrt.cpp +++ b/libavfilter/vsrc_gfxcapture_winrt.cpp @@ -165,6 +165,11 @@ struct GfxCaptureContextCpp { int64_t first_pts { 0 }; int64_t last_pts { 0 }; + + // dup_frames: the last frame sent, and when the next repeat is due (QPC, 100 ns units). + AVFrame *last_frame { nullptr }; + int64_t next_dup { 0 }; + LARGE_INTEGER qpc_freq { }; }; template @@ -978,6 +983,8 @@ static av_cold void gfxcapture_uninit(AVFilterContext *avctx) stop_wgc_thread(avctx); + av_frame_free(&ctx->last_frame); + ctx->d3d.reset(); av_buffer_unref(&ctx->frames_ref); @@ -1394,6 +1401,148 @@ static int render_capture_to_frame(AVFilterContext *avctx, AVFrame *frame, const return 0; } +static void set_frame_props(GfxCaptureContextCpp *ctx, AVFrame *frame) +{ + frame->sample_aspect_ratio = AVRational{1, 1}; + + if (ctx->frames_ctx->sw_format == AV_PIX_FMT_RGBAF16) { + // According to MSDN, all floating point formats contain sRGB image data with linear 1.0 gamma. + frame->color_range = AVCOL_RANGE_JPEG; + frame->color_primaries = AVCOL_PRI_BT709; + frame->color_trc = AVCOL_TRC_LINEAR; + frame->colorspace = AVCOL_SPC_RGB; + } else { + // According to MSDN, all integer formats contain sRGB image data + frame->color_range = AVCOL_RANGE_JPEG; + frame->color_primaries = AVCOL_PRI_BT709; + frame->color_trc = AVCOL_TRC_IEC61966_2_1; + frame->colorspace = AVCOL_SPC_RGB; + } +} + +// Sends a frame whose pts is in QPC 100 ns units (WGC's SystemRelativeTime, or the deadline for a +// repeat) and, with dup_frames, keeps it for the next repeat. +static int output_frame(AVFilterLink *outlink, AVFrame *frame) +{ + AVFilterContext *avctx = outlink->src; + GfxCaptureContext *cctx = CCTX(avctx->priv); + GfxCaptureContextCpp *ctx = cctx->ctx; + int ret; + + // A WGC frame captured just before a repeat's deadline can arrive after the repeat went out + // with a later time: keep the timestamps increasing. + if (cctx->dup_frames && ctx->last_pts && frame->pts <= ctx->last_pts) + frame->pts = ctx->last_pts + 1; + + ctx->last_pts = frame->pts; + + if (!ctx->first_pts) + ctx->first_pts = frame->pts; + frame->pts -= ctx->first_pts; + + if (cctx->dup_frames) { + if (!ctx->last_frame && !(ctx->last_frame = av_frame_alloc())) { + av_frame_free(&frame); + return AVERROR(ENOMEM); + } + ret = av_frame_replace(ctx->last_frame, frame); + if (ret < 0) { + av_frame_free(&frame); + return ret; + } + } + + return ff_filter_frame(outlink, frame); +} + +// The QPC clock in WGC's unit: SystemRelativeTime is QPC time in 100 ns. +static int64_t qpc_now(GfxCaptureContextCpp *ctx) +{ + LARGE_INTEGER now; + if (!ctx->qpc_freq.QuadPart) + QueryPerformanceFrequency(&ctx->qpc_freq); + QueryPerformanceCounter(&now); + return av_rescale(now.QuadPart, TIMESPAN_RES64, ctx->qpc_freq.QuadPart); +} + +// Runs on the WGC thread, like render_capture_to_frame. +static int clear_frame(AVFilterContext *avctx, AVFrame *frame) +{ + GfxCaptureContext *cctx = CCTX(avctx->priv); + GfxCaptureContextCpp *ctx = cctx->ctx; + ID3D11Device *dev = ctx->device_hwctx->device; + ID3D11DeviceContext *dev_ctx = ctx->device_hwctx->device_context; + + D3D11_TEXTURE2D_DESC dst_tex_desc; + reinterpret_cast(frame->data[0])->GetDesc(&dst_tex_desc); + + D3D11_RENDER_TARGET_VIEW_DESC target_desc = {}; + target_desc.Format = dst_tex_desc.Format; + + if (dst_tex_desc.ArraySize > 1) { + target_desc.ViewDimension = D3D11_RTV_DIMENSION_TEXTURE2DARRAY; + target_desc.Texture2DArray.ArraySize = 1; + target_desc.Texture2DArray.FirstArraySlice = (uintptr_t)frame->data[1]; + target_desc.Texture2DArray.MipSlice = 0; + } else { + target_desc.ViewDimension = D3D11_RTV_DIMENSION_TEXTURE2D; + target_desc.Texture2D.MipSlice = 0; + } + + ComPtr rtv; + CHECK_HR_RET(dev->CreateRenderTargetView( + reinterpret_cast(frame->data[0]), &target_desc, &rtv)); + + const float clear_color[4] = {0.f, 0.f, 0.f, 1.f}; + dev_ctx->ClearRenderTargetView(rtv.Get(), clear_color); + + return 0; +} + +// dup_frames: one 1/max_framerate interval in QPC 100 ns units. +static int64_t dup_interval(GfxCaptureContext *cctx) +{ + return av_rescale(TIMESPAN_RES64, cctx->frame_rate.den, cctx->frame_rate.num); +} + +// dup_frames: nothing came from WGC in time. Repeat the last frame (a new reference to the same +// texture), or black before the source delivered any. +static int output_idle_frame(AVFilterLink *outlink) +{ + AVFilterContext *avctx = outlink->src; + GfxCaptureContext *cctx = CCTX(avctx->priv); + GfxCaptureContextCpp *ctx = cctx->ctx; + AVFrame *frame; + + if (ctx->last_frame && ctx->last_frame->buf[0]) { + frame = av_frame_clone(ctx->last_frame); + if (!frame) + return AVERROR(ENOMEM); + } else { + frame = ff_get_video_buffer(outlink, cctx->canvas_width, cctx->canvas_height); + if (!frame) + return AVERROR(ENOMEM); + int ret = run_on_wgc_thread(avctx, [&]() { return clear_frame(avctx, frame); }); + if (ret < 0) { + av_frame_free(&frame); + return ret; + } + set_frame_props(ctx, frame); + } + + // As in ddagrab, the next repeat is due one interval after this one's deadline, not after it + // went out, so timer overshoot does not add up: the repeats a late wakeup delayed follow at once. + // Further behind than a few ticks of the default 15.6 ms timer (50 ms) and more than an interval, + // start over from now instead of catching up. + int64_t now = qpc_now(ctx), interval = dup_interval(cctx); + if (now - ctx->next_dup > FFMAX(interval, TIMESPAN_RES64 / 20)) + ctx->next_dup = now; + frame->pts = FFMIN(ctx->next_dup, now); + ctx->next_dup += interval; + + return output_frame(outlink, frame); +} + static int process_frame_if_exists(AVFilterLink *outlink) { AVFilterContext *avctx = outlink->src; @@ -1434,29 +1583,12 @@ static int process_frame_if_exists(AVFilterLink *outlink) if (ret < 0) return ret; - frame->sample_aspect_ratio = AVRational{1, 1}; - - if (ctx->frames_ctx->sw_format == AV_PIX_FMT_RGBAF16) { - // According to MSDN, all floating point formats contain sRGB image data with linear 1.0 gamma. - frame->color_range = AVCOL_RANGE_JPEG; - frame->color_primaries = AVCOL_PRI_BT709; - frame->color_trc = AVCOL_TRC_LINEAR; - frame->colorspace = AVCOL_SPC_RGB; - } else { - // According to MSDN, all integer formats contain sRGB image data - frame->color_range = AVCOL_RANGE_JPEG; - frame->color_primaries = AVCOL_PRI_BT709; - frame->color_trc = AVCOL_TRC_IEC61966_2_1; - frame->colorspace = AVCOL_SPC_RGB; - } - - ctx->last_pts = frame->pts; + // dup_frames: a new frame restarts the repeat schedule. + if (cctx->dup_frames) + ctx->next_dup = qpc_now(ctx) + dup_interval(cctx); - if (!ctx->first_pts) - ctx->first_pts = frame->pts; - frame->pts -= ctx->first_pts; - - return ff_filter_frame(outlink, frame); + set_frame_props(ctx, frame); + return output_frame(outlink, frame); } static int gfxcapture_activate(AVFilterContext *avctx) @@ -1489,6 +1621,23 @@ static int gfxcapture_activate(AVFilterContext *avctx) break; } + if (cctx->dup_frames) { + int64_t now = qpc_now(ctx); + if (!ctx->next_dup) + ctx->next_dup = now + dup_interval(cctx); + // Like the wait below, never more than 1 s at a time: max_framerate can be far below 1. + int64_t wait = ctx->next_dup - now; + if (wait > 0 && wgctx->frame_arrived_cond.wait_for(frame_lock, + std::chrono::nanoseconds(FFMIN(wait, TIMESPAN_RES64) * 100), [&]() { + return wgctx->frame_seq != last_seq || wgctx->window_closed; + })) + continue; + if (wait > TIMESPAN_RES64) + break; + frame_lock.unlock(); + return output_idle_frame(outlink); + } + if (!wgctx->frame_arrived_cond.wait_for(frame_lock, std::chrono::seconds(1), [&]() { return wgctx->frame_seq != last_seq || wgctx->window_closed; }))