avutil/buffer: Fix race in pool.

This race will always happen sooner or later in a multi-threaded
environment and it will over time lead to OOM.
This fix works by spinning, there are other ways by which this
can be fixed, like simply detecting the issue after it happened
and freeing the over-allocated memory or simply using a mutex.
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavutil/buffer.c

@@ -307,6 +307,7 @@ static AVBufferRef *pool_alloc_buffer(AVBufferPool *pool)
     ret->buffer->free   = pool_release_buffer;
 
     avpriv_atomic_int_add_and_fetch(&pool->refcount, 1);
+    avpriv_atomic_int_add_and_fetch(&pool->nb_allocated, 1);
 
     return ret;
 }
@@ -318,6 +319,12 @@ AVBufferRef *av_buffer_pool_get(AVBufferPool *pool)
 
     /* check whether the pool is empty */
     buf = get_pool(pool);
+    if (!buf && pool->refcount <= pool->nb_allocated) {
+        av_log(NULL, AV_LOG_DEBUG, "Pool race dectected, spining to avoid overallocation and eventual OOM\n");
+        while (!buf && avpriv_atomic_int_get(&pool->refcount) <= avpriv_atomic_int_get(&pool->nb_allocated))
+            buf = get_pool(pool);
+    }
+
     if (!buf)
         return pool_alloc_buffer(pool);
 

libavutil/buffer_internal.h

@@ -85,6 +85,8 @@ struct AVBufferPool {
      */
     volatile int refcount;
 
+    volatile int nb_allocated;
+
     int size;
     AVBufferRef* (*alloc)(int size);
 };