avcodec/dca_parser: Extend DTS core sync word and fix existing check

Check extended sync word for 16-bit LE and BE core streams to reduce
probability of alias sync detection. Previously sync word extension was
checked only for 14-bit streams (and this check did not properly work
across buffer boundary).

Use 64-bit parser state to make extended sync word detection work across
buffer boundary.

This is sufficient to make the sample in ticket #4492 parse
successfully.
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/dca_parser.c

@@ -32,13 +32,20 @@ typedef struct DCAParseContext {
     uint32_t lastmarker;
     int size;
     int framesize;
-    int hd_pos;
 } DCAParseContext;
 
-#define IS_MARKER(state, i, buf, buf_size) \
-    ((state == DCA_SYNCWORD_CORE_14B_LE && (i < buf_size - 2) && (buf[i + 1] & 0xF0) == 0xF0 &&  buf[i + 2]         == 0x07) || \
-     (state == DCA_SYNCWORD_CORE_14B_BE && (i < buf_size - 2) &&  buf[i + 1]         == 0x07 && (buf[i + 2] & 0xF0) == 0xF0) || \
-      state == DCA_SYNCWORD_CORE_LE || state == DCA_SYNCWORD_CORE_BE || state == DCA_SYNCWORD_SUBSTREAM)
+#define IS_CORE_MARKER(state) \
+    (((state & 0xFFFFFFFFF0FF) == (((uint64_t)DCA_SYNCWORD_CORE_14B_LE << 16) | 0xF007)) || \
+     ((state & 0xFFFFFFFFFFF0) == (((uint64_t)DCA_SYNCWORD_CORE_14B_BE << 16) | 0x07F0)) || \
+     ((state & 0xFFFFFFFF00FC) == (((uint64_t)DCA_SYNCWORD_CORE_LE     << 16) | 0x00FC)) || \
+     ((state & 0xFFFFFFFFFC00) == (((uint64_t)DCA_SYNCWORD_CORE_BE     << 16) | 0xFC00)))
+
+#define IS_EXSS_MARKER(state)   ((state & 0xFFFFFFFF) == DCA_SYNCWORD_SUBSTREAM)
+
+#define IS_MARKER(state)    (IS_CORE_MARKER(state) || IS_EXSS_MARKER(state))
+
+#define CORE_MARKER(state)  ((state >> 16) & 0xFFFFFFFF)
+#define EXSS_MARKER(state)  (state & 0xFFFFFFFF)
 
 /**
  * Find the end of the current frame in the bitstream.
@@ -48,20 +55,23 @@ static int dca_find_frame_end(DCAParseContext *pc1, const uint8_t *buf,
                               int buf_size)
 {
     int start_found, i;
-    uint32_t state;
+    uint64_t state;
     ParseContext *pc = &pc1->pc;
 
     start_found = pc->frame_start_found;
-    state       = pc->state;
+    state       = pc->state64;
 
     i = 0;
     if (!start_found) {
         for (i = 0; i < buf_size; i++) {
             state = (state << 8) | buf[i];
-            if (IS_MARKER(state, i, buf, buf_size)) {
-                if (!pc1->lastmarker || state == pc1->lastmarker || pc1->lastmarker == DCA_SYNCWORD_SUBSTREAM) {
+            if (IS_MARKER(state)) {
+                if (!pc1->lastmarker || CORE_MARKER(state) == pc1->lastmarker || pc1->lastmarker == DCA_SYNCWORD_SUBSTREAM) {
                     start_found = 1;
-                    pc1->lastmarker = state;
+                    if (IS_EXSS_MARKER(state))
+                        pc1->lastmarker = EXSS_MARKER(state);
+                    else
+                        pc1->lastmarker = CORE_MARKER(state);
                     i++;
                     break;
                 }
@@ -72,20 +82,18 @@ static int dca_find_frame_end(DCAParseContext *pc1, const uint8_t *buf,
         for (; i < buf_size; i++) {
             pc1->size++;
             state = (state << 8) | buf[i];
-            if (state == DCA_SYNCWORD_SUBSTREAM && !pc1->hd_pos)
-                pc1->hd_pos = pc1->size;
-            if (IS_MARKER(state, i, buf, buf_size) && (state == pc1->lastmarker || pc1->lastmarker == DCA_SYNCWORD_SUBSTREAM)) {
+            if (IS_MARKER(state) && (CORE_MARKER(state) == pc1->lastmarker || pc1->lastmarker == DCA_SYNCWORD_SUBSTREAM)) {
                 if (pc1->framesize > pc1->size)
                     continue;
                 pc->frame_start_found = 0;
-                pc->state             = -1;
+                pc->state64           = -1;
                 pc1->size             = 0;
-                return i - 3;
+                return IS_EXSS_MARKER(state) ? i - 3 : i - 5;
             }
         }
     }
     pc->frame_start_found = start_found;
-    pc->state             = state;
+    pc->state64           = state;
     return END_NOT_FOUND;
 }