Make ff_win32_open more robust.

- Make MultiByteToWideChar fail when it encounters invalid encoding.
  Without this, invalid characters might just be skipped
- When MultiByteToWideChar fails, assume the file name is in CP_ACP
  and open it via normal open function, even when the file will be
  written
- When malloc fails return error instead of crashing
Signed-off-by: Reimar Döffinger <Reimar.Doeffinger@gmx.de>

libavformat/os_support.c

@@ -44,20 +44,23 @@ int ff_win32_open(const char *filename_utf8, int oflag, int pmode)
     wchar_t *filename_w;
 
     /* convert UTF-8 to wide chars */
-    num_chars = MultiByteToWideChar(CP_UTF8, 0, filename_utf8, -1, NULL, 0);
+    num_chars = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, filename_utf8, -1, NULL, 0);
     if (num_chars <= 0)
-        return -1;
+        goto fallback;
     filename_w = av_mallocz(sizeof(wchar_t) * num_chars);
+    if (!filename_w)
+        return -1;
     MultiByteToWideChar(CP_UTF8, 0, filename_utf8, -1, filename_w, num_chars);
 
     fd = _wsopen(filename_w, oflag, SH_DENYNO, pmode);
     av_freep(&filename_w);
 
-    /* filename maybe be in CP_ACP */
-    if (fd == -1 && !(oflag & O_CREAT))
-        return _sopen(filename_utf8, oflag, SH_DENYNO, pmode);
+    if (fd != -1 || (oflag & O_CREAT))
+        return fd;
 
-    return fd;
+fallback:
+    /* filename maybe be in CP_ACP */
+    return _sopen(filename_utf8, oflag, SH_DENYNO, pmode);
 }
 #endif