Commit 2bfa067d authored by Mark Thompson's avatar Mark Thompson

vaapi_encode: Check config attributes before creating config

This prevents attempts to use unsupported modes, such as low-power
H.264 mode on non-Skylake targets.  Also fixes a crash on invalid
configuration, when trying to destroy an invalid VA config/context.
parent 6641819f
......@@ -20,6 +20,7 @@
#include <string.h>
#include "libavutil/avassert.h"
#include "libavutil/common.h"
#include "libavutil/log.h"
#include "libavutil/pixdesc.h"
......@@ -887,6 +888,122 @@ fail:
return err;
}
static av_cold int vaapi_encode_check_config(AVCodecContext *avctx)
{
VAAPIEncodeContext *ctx = avctx->priv_data;
VAStatus vas;
int i, n, err;
VAProfile *profiles = NULL;
VAEntrypoint *entrypoints = NULL;
VAConfigAttrib attr[] = {
{ VAConfigAttribRateControl },
{ VAConfigAttribEncMaxRefFrames },
};
n = vaMaxNumProfiles(ctx->hwctx->display);
profiles = av_malloc_array(n, sizeof(VAProfile));
if (!profiles) {
err = AVERROR(ENOMEM);
goto fail;
}
vas = vaQueryConfigProfiles(ctx->hwctx->display, profiles, &n);
if (vas != VA_STATUS_SUCCESS) {
av_log(ctx, AV_LOG_ERROR, "Failed to query profiles: %d (%s).\n",
vas, vaErrorStr(vas));
err = AVERROR(ENOSYS);
goto fail;
}
for (i = 0; i < n; i++) {
if (profiles[i] == ctx->va_profile)
break;
}
if (i >= n) {
av_log(ctx, AV_LOG_ERROR, "Encoding profile not found (%d).\n",
ctx->va_profile);
err = AVERROR(ENOSYS);
goto fail;
}
n = vaMaxNumEntrypoints(ctx->hwctx->display);
entrypoints = av_malloc_array(n, sizeof(VAEntrypoint));
if (!entrypoints) {
err = AVERROR(ENOMEM);
goto fail;
}
vas = vaQueryConfigEntrypoints(ctx->hwctx->display, ctx->va_profile,
entrypoints, &n);
if (vas != VA_STATUS_SUCCESS) {
av_log(ctx, AV_LOG_ERROR, "Failed to query entrypoints for "
"profile %u: %d (%s).\n", ctx->va_profile,
vas, vaErrorStr(vas));
err = AVERROR(ENOSYS);
goto fail;
}
for (i = 0; i < n; i++) {
if (entrypoints[i] == ctx->va_entrypoint)
break;
}
if (i >= n) {
av_log(ctx, AV_LOG_ERROR, "Encoding entrypoint not found "
"(%d / %d).\n", ctx->va_profile, ctx->va_entrypoint);
err = AVERROR(ENOSYS);
goto fail;
}
vas = vaGetConfigAttributes(ctx->hwctx->display,
ctx->va_profile, ctx->va_entrypoint,
attr, FF_ARRAY_ELEMS(attr));
if (vas != VA_STATUS_SUCCESS) {
av_log(avctx, AV_LOG_ERROR, "Failed to fetch config "
"attributes: %d (%s).\n", vas, vaErrorStr(vas));
return AVERROR(EINVAL);
}
for (i = 0; i < FF_ARRAY_ELEMS(attr); i++) {
if (attr[i].value == VA_ATTRIB_NOT_SUPPORTED) {
// Unfortunately we have to treat this as "don't know" and hope
// for the best, because the Intel MJPEG encoder returns this
// for all the interesting attributes.
continue;
}
switch (attr[i].type) {
case VAConfigAttribRateControl:
if (!(ctx->va_rc_mode & attr[i].value)) {
av_log(avctx, AV_LOG_ERROR, "Rate control mode is not "
"supported: %x\n", attr[i].value);
err = AVERROR(EINVAL);
goto fail;
}
break;
case VAConfigAttribEncMaxRefFrames:
{
unsigned int ref_l0 = attr[i].value & 0xffff;
unsigned int ref_l1 = (attr[i].value >> 16) & 0xffff;
if (avctx->gop_size > 1 && ref_l0 < 1) {
av_log(avctx, AV_LOG_ERROR, "P frames are not "
"supported (%x).\n", attr[i].value);
err = AVERROR(EINVAL);
goto fail;
}
if (avctx->max_b_frames > 0 && ref_l1 < 1) {
av_log(avctx, AV_LOG_ERROR, "B frames are not "
"supported (%x).\n", attr[i].value);
err = AVERROR(EINVAL);
goto fail;
}
}
break;
}
}
err = 0;
fail:
av_freep(&profiles);
av_freep(&entrypoints);
return err;
}
av_cold int ff_vaapi_encode_init(AVCodecContext *avctx,
const VAAPIEncodeType *type)
{
......@@ -907,6 +1024,9 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx,
ctx->codec = type;
ctx->codec_options = ctx->codec_options_data;
ctx->va_config = VA_INVALID_ID;
ctx->va_context = VA_INVALID_ID;
ctx->priv_data = av_mallocz(type->priv_data_size);
if (!ctx->priv_data) {
err = AVERROR(ENOMEM);
......@@ -932,6 +1052,10 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx,
if (err < 0)
goto fail;
err = vaapi_encode_check_config(avctx);
if (err < 0)
goto fail;
vas = vaCreateConfig(ctx->hwctx->display,
ctx->va_profile, ctx->va_entrypoint,
ctx->config_attributes, ctx->nb_config_attributes,
......@@ -1088,11 +1212,15 @@ av_cold int ff_vaapi_encode_close(AVCodecContext *avctx)
vaapi_encode_free(avctx, pic);
}
if (ctx->va_context != VA_INVALID_ID)
if (ctx->va_context != VA_INVALID_ID) {
vaDestroyContext(ctx->hwctx->display, ctx->va_context);
ctx->va_context = VA_INVALID_ID;
}
if (ctx->va_config != VA_INVALID_ID)
if (ctx->va_config != VA_INVALID_ID) {
vaDestroyConfig(ctx->hwctx->display, ctx->va_config);
ctx->va_config = VA_INVALID_ID;
}
if (ctx->codec->close)
ctx->codec->close(avctx);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment