Unreviewed, rolling out r171357.
[WebKit-https.git] / Source / WebCore / platform / graphics / avfoundation / InbandTextTrackPrivateAVF.cpp
1 /*
2  * Copyright (C) 2012-2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #if ENABLE(VIDEO) && (USE(AVFOUNDATION) || PLATFORM(IOS))
29
30 #include "InbandTextTrackPrivateAVF.h"
31
32 #include "ISOVTTCue.h"
33 #include "InbandTextTrackPrivateClient.h"
34 #include "Logging.h"
35 #include "SoftLinking.h"
36 #include <CoreMedia/CoreMedia.h>
37 #include <runtime/ArrayBuffer.h>
38 #include <runtime/DataView.h>
39 #include <runtime/Int8Array.h>
40 #include <wtf/MediaTime.h>
41 #include <wtf/NeverDestroyed.h>
42 #include <wtf/PassOwnPtr.h>
43 #include <wtf/text/CString.h>
44 #include <wtf/text/StringBuilder.h>
45 #include <wtf/text/WTFString.h>
46 #include <wtf/unicode/CharacterNames.h>
47
48 #if !PLATFORM(WIN)
49 #include "MediaTimeMac.h"
50 #endif
51
52 #if !PLATFORM(WIN)
53 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_FRAMEWORK_OPTIONAL(Lib)
54 #define SOFT_LINK_AVF_POINTER(Lib, Name, Type) SOFT_LINK_POINTER_OPTIONAL(Lib, Name, Type)
55 #else
56 #ifdef DEBUG_ALL
57 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_DEBUG_LIBRARY(Lib)
58 #else
59 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_LIBRARY(Lib)
60 #endif
61
62 #define SOFT_LINK_AVF_POINTER(Lib, Name, Type) SOFT_LINK_VARIABLE_DLL_IMPORT_OPTIONAL(Lib, Name, Type)
63 #endif
64
65 SOFT_LINK_AVF_FRAMEWORK(CoreMedia)
66
67 #if !PLATFORM(WIN)
68 SOFT_LINK(CoreMedia, CMSampleBufferGetDataBuffer, CMBlockBufferRef, (CMSampleBufferRef sbuf), (sbuf))
69 SOFT_LINK(CoreMedia, CMBlockBufferCopyDataBytes, OSStatus, (CMBlockBufferRef theSourceBuffer, size_t offsetToData, size_t dataLength, void* destination), (theSourceBuffer, offsetToData, dataLength, destination))
70 SOFT_LINK(CoreMedia, CMBlockBufferGetDataLength, size_t, (CMBlockBufferRef theBuffer), (theBuffer))
71 SOFT_LINK(CoreMedia, CMSampleBufferGetSampleTimingInfo, OSStatus, (CMSampleBufferRef sbuf, CMItemIndex sampleIndex, CMSampleTimingInfo* timingInfoOut), (sbuf, sampleIndex, timingInfoOut))
72 SOFT_LINK(CoreMedia, CMFormatDescriptionGetExtensions, CFDictionaryRef, (CMFormatDescriptionRef desc), (desc))
73 SOFT_LINK(CoreMedia, CMSampleBufferGetFormatDescription, CMFormatDescriptionRef, (CMSampleBufferRef sbuf), (sbuf))
74 #endif
75
76 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_Alignment, CFStringRef)
77 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAlignmentType_Start, CFStringRef)
78 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAlignmentType_Middle, CFStringRef)
79 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAlignmentType_End, CFStringRef)
80 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_BoldStyle, CFStringRef)
81 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_ItalicStyle, CFStringRef)
82 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_UnderlineStyle, CFStringRef)
83 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection, CFStringRef)
84 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_WritingDirectionSizePercentage, CFStringRef)
85 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection, CFStringRef)
86 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_VerticalLayout, CFStringRef)
87 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextVerticalLayout_LeftToRight, CFStringRef)
88 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextVerticalLayout_RightToLeft, CFStringRef)
89 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight, CFStringRef)
90 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_RelativeFontSize, CFStringRef)
91 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_FontFamilyName, CFStringRef)
92 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_ForegroundColorARGB, CFStringRef)
93 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_BackgroundColorARGB, CFStringRef)
94 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_CharacterBackgroundColorARGB, CFStringRef)
95 SOFT_LINK_AVF_POINTER(CoreMedia, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms, CFStringRef)
96
97 #define kCMTextMarkupAttribute_Alignment getkCMTextMarkupAttribute_Alignment()
98 #define kCMTextMarkupAlignmentType_Start getkCMTextMarkupAlignmentType_Start()
99 #define kCMTextMarkupAlignmentType_Middle getkCMTextMarkupAlignmentType_Middle()
100 #define kCMTextMarkupAlignmentType_End getkCMTextMarkupAlignmentType_End()
101 #define kCMTextMarkupAttribute_BoldStyle getkCMTextMarkupAttribute_BoldStyle()
102 #define kCMTextMarkupAttribute_ItalicStyle getkCMTextMarkupAttribute_ItalicStyle()
103 #define kCMTextMarkupAttribute_UnderlineStyle getkCMTextMarkupAttribute_UnderlineStyle()
104 #define kCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection getkCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection()
105 #define kCMTextMarkupAttribute_WritingDirectionSizePercentage getkCMTextMarkupAttribute_WritingDirectionSizePercentage()
106 #define kCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection getkCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection()
107 #define kCMTextMarkupAttribute_VerticalLayout getkCMTextMarkupAttribute_VerticalLayout()
108 #define kCMTextVerticalLayout_LeftToRight getkCMTextVerticalLayout_LeftToRight()
109 #define kCMTextVerticalLayout_RightToLeft getkCMTextVerticalLayout_RightToLeft()
110 #define kCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight getkCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight()
111 #define kCMTextMarkupAttribute_RelativeFontSize getkCMTextMarkupAttribute_RelativeFontSize()
112 #define kCMTextMarkupAttribute_FontFamilyName getkCMTextMarkupAttribute_FontFamilyName()
113 #define kCMTextMarkupAttribute_ForegroundColorARGB getkCMTextMarkupAttribute_ForegroundColorARGB()
114 #define kCMTextMarkupAttribute_BackgroundColorARGB getkCMTextMarkupAttribute_BackgroundColorARGB()
115 #define kCMTextMarkupAttribute_CharacterBackgroundColorARGB getkCMTextMarkupAttribute_CharacterBackgroundColorARGB()
116 #define kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms getkCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms()
117
118 namespace JSC {
119 class ArrayBuffer;
120 }
121
122 namespace WebCore {
123
124 AVFInbandTrackParent::~AVFInbandTrackParent()
125 {
126 }
127
128 InbandTextTrackPrivateAVF::InbandTextTrackPrivateAVF(AVFInbandTrackParent* owner, CueFormat format)
129     : InbandTextTrackPrivate(format)
130     , m_owner(owner)
131     , m_pendingCueStatus(None)
132     , m_index(0)
133     , m_hasBeenReported(false)
134     , m_seeking(false)
135     , m_haveReportedVTTHeader(false)
136 {
137 }
138
139 InbandTextTrackPrivateAVF::~InbandTextTrackPrivateAVF()
140 {
141     disconnect();
142 }
143
144 static bool makeRGBA32FromARGBCFArray(CFArrayRef colorArray, RGBA32& color)
145 {
146     if (CFArrayGetCount(colorArray) < 4)
147         return false;
148
149     float componentArray[4];
150     for (int i = 0; i < 4; i++) {
151         CFNumberRef value = static_cast<CFNumberRef>(CFArrayGetValueAtIndex(colorArray, i));
152         if (CFGetTypeID(value) != CFNumberGetTypeID())
153             return false;
154
155         float component;
156         CFNumberGetValue(value, kCFNumberFloatType, &component);
157         componentArray[i] = component;
158     }
159
160     color = makeRGBA32FromFloats(componentArray[1], componentArray[2], componentArray[3], componentArray[0]);
161     return true;
162 }
163
164 void InbandTextTrackPrivateAVF::processCueAttributes(CFAttributedStringRef attributedString, GenericCueData& cueData)
165 {
166     // Some of the attributes we translate into per-cue WebVTT settings are are repeated on each part of an attributed string so only
167     // process the first instance of each.
168     enum AttributeFlags {
169         Line = 1 << 0,
170         Position = 1 << 1,
171         Size = 1 << 2,
172         Vertical = 1 << 3,
173         Align = 1 << 4,
174         FontName = 1 << 5
175     };
176     unsigned processed = 0;
177
178     StringBuilder content;
179     String attributedStringValue = CFAttributedStringGetString(attributedString);
180     CFIndex length = attributedStringValue.length();
181     if (!length)
182         return;
183
184     CFRange effectiveRange = CFRangeMake(0, 0);
185     while ((effectiveRange.location + effectiveRange.length) < length) {
186
187         CFDictionaryRef attributes = CFAttributedStringGetAttributes(attributedString, effectiveRange.location + effectiveRange.length, &effectiveRange);
188         if (!attributes)
189             continue;
190
191         StringBuilder tagStart;
192         CFStringRef valueString;
193         String tagEnd;
194         CFIndex attributeCount = CFDictionaryGetCount(attributes);
195         Vector<const void*> keys(attributeCount);
196         Vector<const void*> values(attributeCount);
197         CFDictionaryGetKeysAndValues(attributes, keys.data(), values.data());
198
199         for (CFIndex i = 0; i < attributeCount; ++i) {
200             CFStringRef key = static_cast<CFStringRef>(keys[i]);
201             CFTypeRef value = values[i];
202             if (CFGetTypeID(key) != CFStringGetTypeID() || !CFStringGetLength(key))
203                 continue;
204
205             if (CFStringCompare(key, kCMTextMarkupAttribute_Alignment, 0) == kCFCompareEqualTo) {
206                 valueString = static_cast<CFStringRef>(value);
207                 if (CFGetTypeID(valueString) != CFStringGetTypeID() || !CFStringGetLength(valueString))
208                     continue;
209                 if (processed & Align)
210                     continue;
211                 processed |= Align;
212
213                 if (CFStringCompare(valueString, kCMTextMarkupAlignmentType_Start, 0) == kCFCompareEqualTo)
214                     cueData.setAlign(GenericCueData::Start);
215                 else if (CFStringCompare(valueString, kCMTextMarkupAlignmentType_Middle, 0) == kCFCompareEqualTo)
216                     cueData.setAlign(GenericCueData::Middle);
217                 else if (CFStringCompare(valueString, kCMTextMarkupAlignmentType_End, 0) == kCFCompareEqualTo)
218                     cueData.setAlign(GenericCueData::End);
219                 else
220                     ASSERT_NOT_REACHED();
221
222                 continue;
223             }
224
225             if (CFStringCompare(key, kCMTextMarkupAttribute_BoldStyle, 0) == kCFCompareEqualTo) {
226                 if (static_cast<CFBooleanRef>(value) != kCFBooleanTrue)
227                     continue;
228
229                 tagStart.append("<b>");
230                 tagEnd = "</b>" + tagEnd;
231                 continue;
232             }
233
234             if (CFStringCompare(key, kCMTextMarkupAttribute_ItalicStyle, 0) == kCFCompareEqualTo) {
235                 if (static_cast<CFBooleanRef>(value) != kCFBooleanTrue)
236                     continue;
237
238                 tagStart.append("<i>");
239                 tagEnd = "</i>" + tagEnd;
240                 continue;
241             }
242
243             if (CFStringCompare(key, kCMTextMarkupAttribute_UnderlineStyle, 0) == kCFCompareEqualTo) {
244                 if (static_cast<CFBooleanRef>(value) != kCFBooleanTrue)
245                     continue;
246
247                 tagStart.append("<u>");
248                 tagEnd = "</u>" + tagEnd;
249                 continue;
250             }
251
252             if (CFStringCompare(key, kCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection, 0) == kCFCompareEqualTo) {
253                 if (CFGetTypeID(value) != CFNumberGetTypeID())
254                     continue;
255                 if (processed & Line)
256                     continue;
257                 processed |= Line;
258
259                 CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
260                 double line;
261                 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &line);
262                 cueData.setLine(line);
263                 continue;
264             }
265
266             if (CFStringCompare(key, kCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection, 0) == kCFCompareEqualTo) {
267                 if (CFGetTypeID(value) != CFNumberGetTypeID())
268                     continue;
269                 if (processed & Position)
270                     continue;
271                 processed |= Position;
272
273                 CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
274                 double position;
275                 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &position);
276                 cueData.setPosition(position);
277                 continue;
278             }
279
280             if (CFStringCompare(key, kCMTextMarkupAttribute_WritingDirectionSizePercentage, 0) == kCFCompareEqualTo) {
281                 if (CFGetTypeID(value) != CFNumberGetTypeID())
282                     continue;
283                 if (processed & Size)
284                     continue;
285                 processed |= Size;
286
287                 CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
288                 double size;
289                 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &size);
290                 cueData.setSize(size);
291                 continue;
292             }
293
294             if (CFStringCompare(key, kCMTextMarkupAttribute_VerticalLayout, 0) == kCFCompareEqualTo) {
295                 valueString = static_cast<CFStringRef>(value);
296                 if (CFGetTypeID(valueString) != CFStringGetTypeID() || !CFStringGetLength(valueString))
297                     continue;
298                 
299                 if (CFStringCompare(valueString, kCMTextVerticalLayout_LeftToRight, 0) == kCFCompareEqualTo)
300                     tagStart.append(leftToRightMark);
301                 else if (CFStringCompare(valueString, kCMTextVerticalLayout_RightToLeft, 0) == kCFCompareEqualTo)
302                     tagStart.append(rightToLeftMark);
303                 continue;
304             }
305
306             if (CFStringCompare(key, kCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight, 0) == kCFCompareEqualTo) {
307                 if (CFGetTypeID(value) != CFNumberGetTypeID())
308                     continue;
309                 
310                 CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
311                 double baseFontSize;
312                 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &baseFontSize);
313                 cueData.setBaseFontSize(baseFontSize);
314                 continue;
315             }
316
317             if (CFStringCompare(key, kCMTextMarkupAttribute_RelativeFontSize, 0) == kCFCompareEqualTo) {
318                 if (CFGetTypeID(value) != CFNumberGetTypeID())
319                     continue;
320                 
321                 CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
322                 double relativeFontSize;
323                 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &relativeFontSize);
324                 cueData.setRelativeFontSize(relativeFontSize);
325                 continue;
326             }
327
328             if (CFStringCompare(key, kCMTextMarkupAttribute_FontFamilyName, 0) == kCFCompareEqualTo) {
329                 valueString = static_cast<CFStringRef>(value);
330                 if (CFGetTypeID(valueString) != CFStringGetTypeID() || !CFStringGetLength(valueString))
331                     continue;
332                 if (processed & FontName)
333                     continue;
334                 processed |= FontName;
335                 
336                 cueData.setFontName(valueString);
337                 continue;
338             }
339
340             if (CFStringCompare(key, kCMTextMarkupAttribute_ForegroundColorARGB, 0) == kCFCompareEqualTo) {
341                 CFArrayRef arrayValue = static_cast<CFArrayRef>(value);
342                 if (CFGetTypeID(arrayValue) != CFArrayGetTypeID())
343                     continue;
344                 
345                 RGBA32 color;
346                 if (!makeRGBA32FromARGBCFArray(arrayValue, color))
347                     continue;
348                 cueData.setForegroundColor(color);
349             }
350             
351             if (CFStringCompare(key, kCMTextMarkupAttribute_BackgroundColorARGB, 0) == kCFCompareEqualTo) {
352                 CFArrayRef arrayValue = static_cast<CFArrayRef>(value);
353                 if (CFGetTypeID(arrayValue) != CFArrayGetTypeID())
354                     continue;
355                 
356                 RGBA32 color;
357                 if (!makeRGBA32FromARGBCFArray(arrayValue, color))
358                     continue;
359                 cueData.setBackgroundColor(color);
360             }
361
362             if (CFStringCompare(key, kCMTextMarkupAttribute_CharacterBackgroundColorARGB, 0) == kCFCompareEqualTo) {
363                 CFArrayRef arrayValue = static_cast<CFArrayRef>(value);
364                 if (CFGetTypeID(arrayValue) != CFArrayGetTypeID())
365                     continue;
366                 
367                 RGBA32 color;
368                 if (!makeRGBA32FromARGBCFArray(arrayValue, color))
369                     continue;
370                 cueData.setHighlightColor(color);
371             }
372         }
373
374         content.append(tagStart);
375         content.append(attributedStringValue.substring(effectiveRange.location, effectiveRange.length));
376         content.append(tagEnd);
377     }
378
379     if (content.length())
380         cueData.setContent(content.toString());
381 }
382
383 void InbandTextTrackPrivateAVF::processCue(CFArrayRef attributedStrings, CFArrayRef nativeSamples, double time)
384 {
385     if (!client())
386         return;
387
388     processAttributedStrings(attributedStrings, time);
389     processNativeSamples(nativeSamples, time);
390 }
391
392 void InbandTextTrackPrivateAVF::processAttributedStrings(CFArrayRef attributedStrings, double time)
393 {
394     LOG(Media, "InbandTextTrackPrivateAVF::processAttributedStrings - %li attributed strings at time %.2f\n", attributedStrings ? CFArrayGetCount(attributedStrings) : 0, time);
395
396     Vector<RefPtr<GenericCueData>> arrivingCues;
397     if (attributedStrings) {
398         CFIndex count = CFArrayGetCount(attributedStrings);
399         for (CFIndex i = 0; i < count; i++) {
400             CFAttributedStringRef attributedString = static_cast<CFAttributedStringRef>(CFArrayGetValueAtIndex(attributedStrings, i));
401             
402             if (!attributedString || !CFAttributedStringGetLength(attributedString))
403                 continue;
404             
405             RefPtr<GenericCueData> cueData = GenericCueData::create();
406             processCueAttributes(attributedString, *cueData.get());
407             if (!cueData->content().length())
408                 continue;
409             
410             arrivingCues.append(cueData);
411             
412             cueData->setStartTime(time);
413             cueData->setEndTime(std::numeric_limits<double>::infinity());
414             
415             // AVFoundation cue "position" is to the center of the text so adjust relative to the edge because we will use it to
416             // set CSS "left".
417             if (cueData->position() >= 0 && cueData->size() > 0)
418                 cueData->setPosition(cueData->position() - cueData->size() / 2);
419             
420             LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - considering cue (\"%s\") for time = %.2f, position =  %.2f, line =  %.2f", this, cueData->content().utf8().data(), cueData->startTime(), cueData->position(), cueData->line());
421             
422             cueData->setStatus(GenericCueData::Partial);
423         }
424     }
425
426     if (m_pendingCueStatus != None) {
427         // Cues do not have an explicit duration, they are displayed until the next "cue" (which might be empty) is emitted.
428         m_currentCueEndTime = time;
429
430         if (m_currentCueEndTime >= m_currentCueStartTime) {
431             for (auto& cueData : m_cues) {
432                 // See if one of the newly-arrived cues is an extension of this cue.
433                 Vector<RefPtr<GenericCueData>> nonExtensionCues;
434                 for (auto& arrivingCue : arrivingCues) {
435                     if (!arrivingCue->doesExtendCueData(*cueData))
436                         nonExtensionCues.append(arrivingCue);
437                     else
438                         LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - found an extension cue (\"%s\") for time = %.2f, position =  %.2f, line =  %.2f", this, arrivingCue->content().utf8().data(), arrivingCue->startTime(), arrivingCue->position(), arrivingCue->line());
439                 }
440
441                 bool currentCueIsExtended = (arrivingCues.size() != nonExtensionCues.size());
442
443                 arrivingCues = nonExtensionCues;
444                 
445                 if (currentCueIsExtended)
446                     continue;
447
448                 if (m_pendingCueStatus == Valid) {
449                     cueData->setEndTime(m_currentCueEndTime);
450                     cueData->setStatus(GenericCueData::Complete);
451
452                     LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - updating cue: start=%.2f, end=%.2f, content=\"%s\"", this, cueData->startTime(), m_currentCueEndTime, cueData->content().utf8().data());
453                     client()->updateGenericCue(this, cueData.get());
454                 } else {
455                     // We have to assume that the implicit duration is invalid for cues delivered during a seek because the AVF decode pipeline may not
456                     // see every cue, so DO NOT update cue duration while seeking.
457                     LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - ignoring cue delivered during seek: start=%.2f, end=%.2f, content=\"%s\"", this, cueData->startTime(), m_currentCueEndTime, cueData->content().utf8().data());
458                 }
459             }
460         } else
461             LOG(Media, "InbandTextTrackPrivateAVF::processCue negative length cue(s) ignored: start=%.2f, end=%.2f\n", m_currentCueStartTime, m_currentCueEndTime);
462
463         removeCompletedCues();
464     }
465
466     if (arrivingCues.isEmpty())
467         return;
468
469     m_currentCueStartTime = time;
470
471     for (auto& cueData : arrivingCues) {
472
473         m_cues.append(cueData);
474         
475         LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - adding cue for time = %.2f, position =  %.2f, line =  %.2f", this, cueData->startTime(), cueData->position(), cueData->line());
476
477         client()->addGenericCue(this, cueData.release());
478     }
479
480     m_pendingCueStatus = seeking() ? DeliveredDuringSeek : Valid;
481 }
482
483 void InbandTextTrackPrivateAVF::beginSeeking()
484 {
485     // Forget any partially accumulated cue data as the seek could be to a time outside of the cue's
486     // range, which will mean that the next cue delivered will result in the current cue getting the
487     // incorrect duration.
488     resetCueValues();
489     m_seeking = true;
490 }
491
492 void InbandTextTrackPrivateAVF::disconnect()
493 {
494     m_owner = 0;
495     m_index = 0;
496 }
497
498 void InbandTextTrackPrivateAVF::removeCompletedCues()
499 {
500     if (client()) {
501         long currentCue = m_cues.size() - 1;
502         for (; currentCue >= 0; --currentCue) {
503             if (m_cues[currentCue]->status() != GenericCueData::Complete)
504                 continue;
505
506             m_cues.remove(currentCue);
507         }
508     }
509
510     if (m_cues.isEmpty())
511         m_pendingCueStatus = None;
512
513     m_currentCueStartTime = 0;
514     m_currentCueEndTime = 0;
515 }
516
517 void InbandTextTrackPrivateAVF::resetCueValues()
518 {
519     if (m_currentCueEndTime && m_cues.size())
520         LOG(Media, "InbandTextTrackPrivateAVF::resetCueValues flushing data for cues: start=%.2f\n", m_currentCueStartTime);
521
522     if (client()) {
523         for (size_t i = 0; i < m_cues.size(); i++)
524             client()->removeGenericCue(this, m_cues[i].get());
525     }
526
527     m_cues.resize(0);
528     m_pendingCueStatus = None;
529     m_currentCueStartTime = 0;
530     m_currentCueEndTime = 0;
531 }
532
533 void InbandTextTrackPrivateAVF::setMode(InbandTextTrackPrivate::Mode newMode)
534 {
535     if (!m_owner)
536         return;
537
538     InbandTextTrackPrivate::Mode oldMode = mode();
539     InbandTextTrackPrivate::setMode(newMode);
540
541     if (oldMode == newMode)
542         return;
543
544     m_owner->trackModeChanged();
545 }
546
547 void InbandTextTrackPrivateAVF::processNativeSamples(CFArrayRef nativeSamples, double presentationTime)
548 {
549 #if PLATFORM(WIN)
550     UNUSED_PARAM(nativeSamples);
551     UNUSED_PARAM(presentationTime);
552     ASSERT_NOT_REACHED();
553 #else
554     if (!nativeSamples)
555         return;
556
557     CFIndex count = CFArrayGetCount(nativeSamples);
558     if (!count)
559         return;
560
561     LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples - %li sample buffers at time %.2f\n", count, presentationTime);
562
563     for (CFIndex i = 0; i < count; i++) {
564
565         CMSampleBufferRef sampleBuffer = (CMSampleBufferRef)CFArrayGetValueAtIndex(nativeSamples, i);
566         if (!sampleBuffer)
567             continue;
568
569         CMSampleTimingInfo timingInfo;
570         OSStatus status = CMSampleBufferGetSampleTimingInfo(sampleBuffer, i, &timingInfo);
571         if (status) {
572             LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - CMSampleBufferGetSampleTimingInfo returned error %x for sample %li", this, static_cast<int>(status), i);
573             continue;
574         }
575
576         CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
577         size_t bufferLength = CMBlockBufferGetDataLength(blockBuffer);
578         if (bufferLength < ISOBox::boxHeaderSize()) {
579             LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - ERROR: CMSampleBuffer size length unexpectedly small (%zu)!!", this, bufferLength);
580             continue;
581         }
582
583         m_sampleInputBuffer.resize(m_sampleInputBuffer.size() + bufferLength);
584         CMBlockBufferCopyDataBytes(blockBuffer, 0, bufferLength, m_sampleInputBuffer.data() + m_sampleInputBuffer.size() - bufferLength);
585
586         RefPtr<ArrayBuffer> buffer = ArrayBuffer::create(m_sampleInputBuffer.data(), m_sampleInputBuffer.size());
587
588         String type = ISOBox::peekType(buffer.get());
589         size_t boxLength = ISOBox::peekLength(buffer.get());
590         if (boxLength > buffer->byteLength()) {
591             LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - ERROR: chunk '%s' size (%zu) larger than buffer length (%u)!!", this, type.utf8().data(), boxLength, buffer->byteLength());
592             continue;
593         }
594
595         LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - chunk type = '%s', size = %zu", this, type.utf8().data(), boxLength);
596
597         if (type == ISOWebVTTCue::boxType()) {
598 #if !PLATFORM(WIN)
599             ISOWebVTTCue cueData = ISOWebVTTCue(MediaTime::createWithDouble(presentationTime), toMediaTime(timingInfo.duration), buffer.get());
600 #else
601             ISOWebVTTCue cueData = ISOWebVTTCue(MediaTime::createWithDouble(presentationTime), MediaTime::createWithDouble(CMTimeGetSeconds(timingInfo.duration)), buffer.get());
602 #endif
603             LOG(Media, "    sample presentation time = %.2f, duration = %.2f", cueData.presentationTime().toDouble(), cueData.duration().toDouble());
604             LOG(Media, "    id = \"%s\", settings = \"%s\", cue text = \"%s\"", cueData.id().utf8().data(), cueData.settings().utf8().data(), cueData.cueText().utf8().data());
605             LOG(Media, "    sourceID = \"%s\", originalStartTime = \"%s\"", cueData.sourceID().utf8().data(), cueData.originalStartTime().utf8().data());
606
607             client()->parseWebVTTCueData(this, cueData);
608         }
609
610         do {
611             if (m_haveReportedVTTHeader)
612                 break;
613
614             CMFormatDescriptionRef formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer);
615             if (!formatDescription)
616                 break;
617
618             CFDictionaryRef extensions = CMFormatDescriptionGetExtensions(formatDescription);
619             if (!extensions)
620                 break;
621
622             CFDictionaryRef sampleDescriptionExtensions = static_cast<CFDictionaryRef>(CFDictionaryGetValue(extensions, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms));
623             if (!sampleDescriptionExtensions)
624                 break;
625             
626             CFDataRef webvttHeaderData = static_cast<CFDataRef>(CFDictionaryGetValue(sampleDescriptionExtensions, CFSTR("vttC")));
627             if (!webvttHeaderData)
628                 break;
629
630             unsigned length = CFDataGetLength(webvttHeaderData);
631             if (!length)
632                 break;
633
634             // A WebVTT header is terminated by "One or more WebVTT line terminators" so append two line feeds to make sure the parser
635             // reccognized this string as a full header.
636             StringBuilder header;
637             header.append(reinterpret_cast<const unsigned char*>(CFDataGetBytePtr(webvttHeaderData)), length);
638             header.append("\n\n");
639
640             LOG(Media, "    vtt header = \n%s", header.toString().utf8().data());
641             client()->parseWebVTTFileHeader(this, header.toString());
642             m_haveReportedVTTHeader = true;
643         } while (0);
644
645         m_sampleInputBuffer.remove(0, boxLength);
646     }
647 #endif
648 }
649
650 } // namespace WebCore
651
652 #endif // ENABLE(VIDEO) && (USE(AVFOUNDATION) || PLATFORM(IOS))