diff --git a/doc/filters.texi b/doc/filters.texi index 3534c87..8e4aa3e 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -30056,6 +30056,14 @@ Crop this many pixels from the left side of the captured frames. If set to 1, return frames with premultiplied alpha. Default is 0 (straight alpha). +@item dup_frames +If set to 1, repeat the last frame when the source delivers no new one within +one interval of @option{max_framerate}, and output black frames until the +source delivers its first one (so any source whose first frame takes longer +than one interval starts with a black frame). Windows Graphics Capture delivers a frame only +when the content changes, so a still source otherwise outputs nothing and holds +back anything muxed with it. Default is 0. + @item resize_mode Defines how the captured content is fitted into the output canvas size. Possible values: diff --git a/libavfilter/vsrc_gfxcapture.c b/libavfilter/vsrc_gfxcapture.c index 19d8a8a..10943d0 100644 --- a/libavfilter/vsrc_gfxcapture.c +++ b/libavfilter/vsrc_gfxcapture.c @@ -57,6 +57,8 @@ static const AVOption gfxcapture_options[] = { { "crop_bottom", "number of pixels to crop from the bottom of the captured area", OFFSET(crop_bottom), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS }, { "premultiplied", "return premultiplied alpha frames", OFFSET(premult_alpha), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS }, + { "dup_frames", "repeat the last frame when the source sends none in time", + OFFSET(dup_frames), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS }, { "resize_mode", "capture source resize behavior", OFFSET(resize_mode), AV_OPT_TYPE_INT, { .i64 = GFX_RESIZE_CROP }, 0, GFX_RESIZE_NB - 1, FLAGS, .unit = "resize_mode" }, { "crop", "crop or add black bars into frame", 0, AV_OPT_TYPE_CONST, { .i64 = GFX_RESIZE_CROP }, 0, 0, FLAGS, .unit = "resize_mode" }, { "scale", "scale source to fit initial size", 0, AV_OPT_TYPE_CONST, { .i64 = GFX_RESIZE_SCALE }, 0, 0, FLAGS, .unit = "resize_mode" }, diff --git a/libavfilter/vsrc_gfxcapture.h b/libavfilter/vsrc_gfxcapture.h index f541f9b..bace2be 100644 --- a/libavfilter/vsrc_gfxcapture.h +++ b/libavfilter/vsrc_gfxcapture.h @@ -67,6 +67,7 @@ typedef struct GfxCaptureContext { int resize_mode; int scale_mode; int premult_alpha; + int dup_frames; } GfxCaptureContext; #ifdef __cplusplus diff --git a/libavfilter/vsrc_gfxcapture_winrt.cpp b/libavfilter/vsrc_gfxcapture_winrt.cpp index ddc6a7d..96ba0ce 100644 --- a/libavfilter/vsrc_gfxcapture_winrt.cpp +++ b/libavfilter/vsrc_gfxcapture_winrt.cpp @@ -48,6 +48,7 @@ extern "C" { } #include +#include #include #include #include @@ -165,6 +166,13 @@ struct GfxCaptureContextCpp { int64_t first_pts { 0 }; int64_t last_pts { 0 }; + + // dup_frames: the last frame sent, and when it went out. + AVFrame *last_frame { nullptr }; + std::chrono::steady_clock::time_point last_output { }; + bool have_output_time { false }; + LARGE_INTEGER qpc_freq { }; + bool logged_clock { false }; }; template @@ -978,6 +986,8 @@ static av_cold void gfxcapture_uninit(AVFilterContext *avctx) stop_wgc_thread(avctx); + av_frame_free(&ctx->last_frame); + ctx->d3d.reset(); av_buffer_unref(&ctx->frames_ref); @@ -1394,6 +1404,135 @@ static int render_capture_to_frame(AVFilterContext *avctx, AVFrame *frame, const return 0; } +static void set_frame_props(GfxCaptureContextCpp *ctx, AVFrame *frame) +{ + frame->sample_aspect_ratio = AVRational{1, 1}; + + if (ctx->frames_ctx->sw_format == AV_PIX_FMT_RGBAF16) { + // According to MSDN, all floating point formats contain sRGB image data with linear 1.0 gamma. + frame->color_range = AVCOL_RANGE_JPEG; + frame->color_primaries = AVCOL_PRI_BT709; + frame->color_trc = AVCOL_TRC_LINEAR; + frame->colorspace = AVCOL_SPC_RGB; + } else { + // According to MSDN, all integer formats contain sRGB image data + frame->color_range = AVCOL_RANGE_JPEG; + frame->color_primaries = AVCOL_PRI_BT709; + frame->color_trc = AVCOL_TRC_IEC61966_2_1; + frame->colorspace = AVCOL_SPC_RGB; + } +} + +// Sends a frame whose pts is in QPC 100 ns units (WGC's SystemRelativeTime, or now for a repeat) and, +// with dup_frames, keeps it for the next repeat. +static int output_frame(AVFilterLink *outlink, AVFrame *frame) +{ + AVFilterContext *avctx = outlink->src; + GfxCaptureContext *cctx = CCTX(avctx->priv); + GfxCaptureContextCpp *ctx = cctx->ctx; + int ret; + + // A WGC frame captured just before a repeat's deadline can arrive after the repeat went out + // with a later time: keep the timestamps increasing. + if (cctx->dup_frames && ctx->last_pts && frame->pts <= ctx->last_pts) + frame->pts = ctx->last_pts + 1; + + ctx->last_pts = frame->pts; + + if (!ctx->first_pts) + ctx->first_pts = frame->pts; + frame->pts -= ctx->first_pts; + + if (cctx->dup_frames) { + if (!ctx->last_frame && !(ctx->last_frame = av_frame_alloc())) { + av_frame_free(&frame); + return AVERROR(ENOMEM); + } + ret = av_frame_replace(ctx->last_frame, frame); + if (ret < 0) { + av_frame_free(&frame); + return ret; + } + ctx->last_output = std::chrono::steady_clock::now(); + ctx->have_output_time = true; + } + + return ff_filter_frame(outlink, frame); +} + +// The QPC clock in WGC's unit: SystemRelativeTime is QPC time in 100 ns. +static int64_t qpc_now(GfxCaptureContextCpp *ctx) +{ + LARGE_INTEGER now; + if (!ctx->qpc_freq.QuadPart) + QueryPerformanceFrequency(&ctx->qpc_freq); + QueryPerformanceCounter(&now); + return av_rescale(now.QuadPart, TIMESPAN_RES64, ctx->qpc_freq.QuadPart); +} + +// Runs on the WGC thread, like render_capture_to_frame. +static int clear_frame(AVFilterContext *avctx, AVFrame *frame) +{ + GfxCaptureContext *cctx = CCTX(avctx->priv); + GfxCaptureContextCpp *ctx = cctx->ctx; + ID3D11Device *dev = ctx->device_hwctx->device; + ID3D11DeviceContext *dev_ctx = ctx->device_hwctx->device_context; + + D3D11_TEXTURE2D_DESC dst_tex_desc; + reinterpret_cast(frame->data[0])->GetDesc(&dst_tex_desc); + + D3D11_RENDER_TARGET_VIEW_DESC target_desc = {}; + target_desc.Format = dst_tex_desc.Format; + + if (dst_tex_desc.ArraySize > 1) { + target_desc.ViewDimension = D3D11_RTV_DIMENSION_TEXTURE2DARRAY; + target_desc.Texture2DArray.ArraySize = 1; + target_desc.Texture2DArray.FirstArraySlice = (uintptr_t)frame->data[1]; + target_desc.Texture2DArray.MipSlice = 0; + } else { + target_desc.ViewDimension = D3D11_RTV_DIMENSION_TEXTURE2D; + target_desc.Texture2D.MipSlice = 0; + } + + ComPtr rtv; + CHECK_HR_RET(dev->CreateRenderTargetView( + reinterpret_cast(frame->data[0]), &target_desc, &rtv)); + + const float clear_color[4] = {0.f, 0.f, 0.f, 1.f}; + dev_ctx->ClearRenderTargetView(rtv.Get(), clear_color); + + return 0; +} + +// dup_frames: nothing came from WGC in time. Repeat the last frame (a new reference to the same +// texture), or black before the source delivered any. +static int output_idle_frame(AVFilterLink *outlink) +{ + AVFilterContext *avctx = outlink->src; + GfxCaptureContext *cctx = CCTX(avctx->priv); + GfxCaptureContextCpp *ctx = cctx->ctx; + AVFrame *frame; + + if (ctx->last_frame && ctx->last_frame->buf[0]) { + frame = av_frame_clone(ctx->last_frame); + if (!frame) + return AVERROR(ENOMEM); + } else { + frame = ff_get_video_buffer(outlink, cctx->canvas_width, cctx->canvas_height); + if (!frame) + return AVERROR(ENOMEM); + int ret = run_on_wgc_thread(avctx, [&]() { return clear_frame(avctx, frame); }); + if (ret < 0) { + av_frame_free(&frame); + return ret; + } + set_frame_props(ctx, frame); + } + + frame->pts = qpc_now(ctx); + return output_frame(outlink, frame); +} + static int process_frame_if_exists(AVFilterLink *outlink) { AVFilterContext *avctx = outlink->src; @@ -1434,29 +1573,16 @@ static int process_frame_if_exists(AVFilterLink *outlink) if (ret < 0) return ret; - frame->sample_aspect_ratio = AVRational{1, 1}; - - if (ctx->frames_ctx->sw_format == AV_PIX_FMT_RGBAF16) { - // According to MSDN, all floating point formats contain sRGB image data with linear 1.0 gamma. - frame->color_range = AVCOL_RANGE_JPEG; - frame->color_primaries = AVCOL_PRI_BT709; - frame->color_trc = AVCOL_TRC_LINEAR; - frame->colorspace = AVCOL_SPC_RGB; - } else { - // According to MSDN, all integer formats contain sRGB image data - frame->color_range = AVCOL_RANGE_JPEG; - frame->color_primaries = AVCOL_PRI_BT709; - frame->color_trc = AVCOL_TRC_IEC61966_2_1; - frame->colorspace = AVCOL_SPC_RGB; + // dup_frames stamps repeats with QPC now: a source clock away from it would bump frames + // (output_frame), so log how far the first source frame is from QPC now. + if (cctx->dup_frames && !ctx->logged_clock) { + ctx->logged_clock = true; + av_log(avctx, AV_LOG_VERBOSE, "First source frame is %" PRId64 " us before QPC now\n", + (qpc_now(ctx) - frame->pts) / 10); } - ctx->last_pts = frame->pts; - - if (!ctx->first_pts) - ctx->first_pts = frame->pts; - frame->pts -= ctx->first_pts; - - return ff_filter_frame(outlink, frame); + set_frame_props(ctx, frame); + return output_frame(outlink, frame); } static int gfxcapture_activate(AVFilterContext *avctx) @@ -1489,6 +1615,21 @@ static int gfxcapture_activate(AVFilterContext *avctx) break; } + if (cctx->dup_frames) { + if (!ctx->have_output_time) { + ctx->last_output = std::chrono::steady_clock::now(); + ctx->have_output_time = true; + } + auto interval = std::chrono::nanoseconds(av_rescale(1000000000, cctx->frame_rate.den, cctx->frame_rate.num)); + if (!wgctx->frame_arrived_cond.wait_until(frame_lock, ctx->last_output + interval, [&]() { + return wgctx->frame_seq != last_seq || wgctx->window_closed; + })) { + frame_lock.unlock(); + return output_idle_frame(outlink); + } + continue; + } + if (!wgctx->frame_arrived_cond.wait_for(frame_lock, std::chrono::seconds(1), [&]() { return wgctx->frame_seq != last_seq || wgctx->window_closed; }))