[Mac] process raw VTT in-band captions
[WebKit-https.git] / Source / WebCore / platform / graphics / avfoundation / InbandTextTrackPrivateAVF.cpp
1 /*
2  * Copyright (C) 2012-2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #if ENABLE(VIDEO) && (USE(AVFOUNDATION) || PLATFORM(IOS))
29
30 #include "InbandTextTrackPrivateAVF.h"
31
32 #include "ISOVTTCue.h"
33 #include "InbandTextTrackPrivateClient.h"
34 #include "Logging.h"
35 #include "SoftLinking.h"
36 #include <CoreMedia/CoreMedia.h>
37 #include <runtime/ArrayBuffer.h>
38 #include <runtime/DataView.h>
39 #include <runtime/Int8Array.h>
40 #include <wtf/MediaTime.h>
41 #include <wtf/NeverDestroyed.h>
42 #include <wtf/PassOwnPtr.h>
43 #include <wtf/text/CString.h>
44 #include <wtf/text/StringBuilder.h>
45 #include <wtf/text/WTFString.h>
46 #include <wtf/unicode/CharacterNames.h>
47
48 #if !PLATFORM(WIN)
49 #include "MediaTimeMac.h"
50 #endif
51
52 #if !PLATFORM(WIN)
53 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_FRAMEWORK_OPTIONAL(Lib)
54 #define SOFT_LINK_AVF_POINTER(Lib, Name, Type) SOFT_LINK_POINTER_OPTIONAL(Lib, Name, Type)
55 #else
56 #ifdef DEBUG_ALL
57 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_DEBUG_LIBRARY(Lib)
58 #else
59 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_LIBRARY(Lib)
60 #endif
61
62 #define SOFT_LINK_AVF_POINTER(Lib, Name, Type) SOFT_LINK_VARIABLE_DLL_IMPORT_OPTIONAL(Lib, Name, Type)
63 #endif
64
65 SOFT_LINK_AVF_FRAMEWORK(CoreMedia)
66
67 #if !PLATFORM(WIN)
68 SOFT_LINK(CoreMedia, CMSampleBufferGetDataBuffer, CMBlockBufferRef, (CMSampleBufferRef sbuf), (sbuf))
69 SOFT_LINK(CoreMedia, CMBlockBufferCopyDataBytes, OSStatus, (CMBlockBufferRef theSourceBuffer, size_t offsetToData, size_t dataLength, void* destination), (theSourceBuffer, offsetToData, dataLength, destination))
70 SOFT_LINK(CoreMedia, CMBlockBufferGetDataLength, size_t, (CMBlockBufferRef theBuffer), (theBuffer))
71 SOFT_LINK(CoreMedia, CMSampleBufferGetSampleTimingInfo, OSStatus, (CMSampleBufferRef sbuf, CMItemIndex sampleIndex, CMSampleTimingInfo* timingInfoOut), (sbuf, sampleIndex, timingInfoOut))
72 SOFT_LINK(CoreMedia, CMFormatDescriptionGetExtensions, CFDictionaryRef, (CMFormatDescriptionRef desc), (desc))
73 SOFT_LINK(CoreMedia, CMSampleBufferGetFormatDescription, CMFormatDescriptionRef, (CMSampleBufferRef sbuf), (sbuf))
74 #else
75
76 SOFT_LINK_DLL_IMPORT(CoreMedia, CMTimeGetSeconds, Float64, __cdecl, (CMTime time), (time))
77 #define CMTimeGetSeconds softLink_CMTimeGetSeconds
78 SOFT_LINK_DLL_IMPORT(CoreMedia, CMSampleBufferGetDataBuffer, CMBlockBufferRef, __cdecl, (CMSampleBufferRef sbuf), (sbuf))
79 #define CMSampleBufferGetDataBuffer softLink_CMSampleBufferGetDataBuffer
80 SOFT_LINK_DLL_IMPORT(CoreMedia, CMBlockBufferCopyDataBytes, OSStatus, __cdecl, (CMBlockBufferRef theSourceBuffer, size_t offsetToData, size_t dataLength, void* destination), (theSourceBuffer, offsetToData, dataLength, destination))
81 #define CMBlockBufferCopyDataBytes softLink_CMBlockBufferCopyDataBytes
82 SOFT_LINK_DLL_IMPORT(CoreMedia, CMBlockBufferGetDataLength, size_t, __cdecl, (CMBlockBufferRef theBuffer), (theBuffer))
83 #define CMBlockBufferGetDataLength softLink_CMBlockBufferGetDataLength
84 SOFT_LINK_DLL_IMPORT(CoreMedia, CMSampleBufferGetSampleTimingInfo, OSStatus, __cdecl, (CMSampleBufferRef sbuf, CMItemIndex sampleIndex, CMSampleTimingInfo* timingInfoOut), (sbuf, sampleIndex, timingInfoOut))
85 #define CMSampleBufferGetSampleTimingInfo softLink_CMSampleBufferGetSampleTimingInfo
86 SOFT_LINK_DLL_IMPORT(CoreMedia, CMFormatDescriptionGetExtensions, CFDictionaryRef, __cdecl, (CMFormatDescriptionRef desc), (desc))
87 #define CMFormatDescriptionGetExtensions softLink_CMFormatDescriptionGetExtensions
88 SOFT_LINK_DLL_IMPORT(CoreMedia, CMSampleBufferGetFormatDescription, CMFormatDescriptionRef, __cdecl, (CMSampleBufferRef sbuf), (sbuf))
89 #define CMSampleBufferGetFormatDescription softLink_CMSampleBufferGetFormatDescription
90
91 #endif
92
93 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_Alignment, CFStringRef)
94 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAlignmentType_Start, CFStringRef)
95 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAlignmentType_Middle, CFStringRef)
96 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAlignmentType_End, CFStringRef)
97 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_BoldStyle, CFStringRef)
98 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_ItalicStyle, CFStringRef)
99 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_UnderlineStyle, CFStringRef)
100 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection, CFStringRef)
101 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_WritingDirectionSizePercentage, CFStringRef)
102 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection, CFStringRef)
103 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_VerticalLayout, CFStringRef)
104 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextVerticalLayout_LeftToRight, CFStringRef)
105 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextVerticalLayout_RightToLeft, CFStringRef)
106 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight, CFStringRef)
107 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_RelativeFontSize, CFStringRef)
108 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_FontFamilyName, CFStringRef)
109 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_ForegroundColorARGB, CFStringRef)
110 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_BackgroundColorARGB, CFStringRef)
111 SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_CharacterBackgroundColorARGB, CFStringRef)
112 SOFT_LINK_AVF_POINTER(CoreMedia, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms, CFStringRef)
113
114 #define kCMTextMarkupAttribute_Alignment getkCMTextMarkupAttribute_Alignment()
115 #define kCMTextMarkupAlignmentType_Start getkCMTextMarkupAlignmentType_Start()
116 #define kCMTextMarkupAlignmentType_Middle getkCMTextMarkupAlignmentType_Middle()
117 #define kCMTextMarkupAlignmentType_End getkCMTextMarkupAlignmentType_End()
118 #define kCMTextMarkupAttribute_BoldStyle getkCMTextMarkupAttribute_BoldStyle()
119 #define kCMTextMarkupAttribute_ItalicStyle getkCMTextMarkupAttribute_ItalicStyle()
120 #define kCMTextMarkupAttribute_UnderlineStyle getkCMTextMarkupAttribute_UnderlineStyle()
121 #define kCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection getkCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection()
122 #define kCMTextMarkupAttribute_WritingDirectionSizePercentage getkCMTextMarkupAttribute_WritingDirectionSizePercentage()
123 #define kCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection getkCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection()
124 #define kCMTextMarkupAttribute_VerticalLayout getkCMTextMarkupAttribute_VerticalLayout()
125 #define kCMTextVerticalLayout_LeftToRight getkCMTextVerticalLayout_LeftToRight()
126 #define kCMTextVerticalLayout_RightToLeft getkCMTextVerticalLayout_RightToLeft()
127 #define kCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight getkCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight()
128 #define kCMTextMarkupAttribute_RelativeFontSize getkCMTextMarkupAttribute_RelativeFontSize()
129 #define kCMTextMarkupAttribute_FontFamilyName getkCMTextMarkupAttribute_FontFamilyName()
130 #define kCMTextMarkupAttribute_ForegroundColorARGB getkCMTextMarkupAttribute_ForegroundColorARGB()
131 #define kCMTextMarkupAttribute_BackgroundColorARGB getkCMTextMarkupAttribute_BackgroundColorARGB()
132 #define kCMTextMarkupAttribute_CharacterBackgroundColorARGB getkCMTextMarkupAttribute_CharacterBackgroundColorARGB()
133 #define kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms getkCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms()
134
135 namespace JSC {
136 class ArrayBuffer;
137 }
138
139 namespace WebCore {
140
141 AVFInbandTrackParent::~AVFInbandTrackParent()
142 {
143 }
144
145 InbandTextTrackPrivateAVF::InbandTextTrackPrivateAVF(AVFInbandTrackParent* owner, CueFormat format)
146     : InbandTextTrackPrivate(format)
147     , m_owner(owner)
148     , m_pendingCueStatus(None)
149     , m_index(0)
150     , m_hasBeenReported(false)
151     , m_seeking(false)
152     , m_haveReportedVTTHeader(false)
153 {
154 }
155
156 InbandTextTrackPrivateAVF::~InbandTextTrackPrivateAVF()
157 {
158     disconnect();
159 }
160
161 static bool makeRGBA32FromARGBCFArray(CFArrayRef colorArray, RGBA32& color)
162 {
163     if (CFArrayGetCount(colorArray) < 4)
164         return false;
165
166     float componentArray[4];
167     for (int i = 0; i < 4; i++) {
168         CFNumberRef value = static_cast<CFNumberRef>(CFArrayGetValueAtIndex(colorArray, i));
169         if (CFGetTypeID(value) != CFNumberGetTypeID())
170             return false;
171
172         float component;
173         CFNumberGetValue(value, kCFNumberFloatType, &component);
174         componentArray[i] = component;
175     }
176
177     color = makeRGBA32FromFloats(componentArray[1], componentArray[2], componentArray[3], componentArray[0]);
178     return true;
179 }
180
181 void InbandTextTrackPrivateAVF::processCueAttributes(CFAttributedStringRef attributedString, GenericCueData* cueData)
182 {
183     // Some of the attributes we translate into per-cue WebVTT settings are are repeated on each part of an attributed string so only
184     // process the first instance of each.
185     enum AttributeFlags {
186         Line = 1 << 0,
187         Position = 1 << 1,
188         Size = 1 << 2,
189         Vertical = 1 << 3,
190         Align = 1 << 4,
191         FontName = 1 << 5
192     };
193     unsigned processed = 0;
194
195     StringBuilder content;
196     String attributedStringValue = CFAttributedStringGetString(attributedString);
197     CFIndex length = attributedStringValue.length();
198     if (!length)
199         return;
200
201     CFRange effectiveRange = CFRangeMake(0, 0);
202     while ((effectiveRange.location + effectiveRange.length) < length) {
203
204         CFDictionaryRef attributes = CFAttributedStringGetAttributes(attributedString, effectiveRange.location + effectiveRange.length, &effectiveRange);
205         if (!attributes)
206             continue;
207
208         StringBuilder tagStart;
209         CFStringRef valueString;
210         String tagEnd;
211         CFIndex attributeCount = CFDictionaryGetCount(attributes);
212         Vector<const void*> keys(attributeCount);
213         Vector<const void*> values(attributeCount);
214         CFDictionaryGetKeysAndValues(attributes, keys.data(), values.data());
215
216         for (CFIndex i = 0; i < attributeCount; ++i) {
217             CFStringRef key = static_cast<CFStringRef>(keys[i]);
218             CFTypeRef value = values[i];
219             if (CFGetTypeID(key) != CFStringGetTypeID() || !CFStringGetLength(key))
220                 continue;
221
222             if (CFStringCompare(key, kCMTextMarkupAttribute_Alignment, 0) == kCFCompareEqualTo) {
223                 valueString = static_cast<CFStringRef>(value);
224                 if (CFGetTypeID(valueString) != CFStringGetTypeID() || !CFStringGetLength(valueString))
225                     continue;
226                 if (processed & Align)
227                     continue;
228                 processed |= Align;
229
230                 if (CFStringCompare(valueString, kCMTextMarkupAlignmentType_Start, 0) == kCFCompareEqualTo)
231                     cueData->setAlign(GenericCueData::Start);
232                 else if (CFStringCompare(valueString, kCMTextMarkupAlignmentType_Middle, 0) == kCFCompareEqualTo)
233                     cueData->setAlign(GenericCueData::Middle);
234                 else if (CFStringCompare(valueString, kCMTextMarkupAlignmentType_End, 0) == kCFCompareEqualTo)
235                     cueData->setAlign(GenericCueData::End);
236                 else
237                     ASSERT_NOT_REACHED();
238
239                 continue;
240             }
241
242             if (CFStringCompare(key, kCMTextMarkupAttribute_BoldStyle, 0) == kCFCompareEqualTo) {
243                 if (static_cast<CFBooleanRef>(value) != kCFBooleanTrue)
244                     continue;
245
246                 tagStart.append("<b>");
247                 tagEnd = "</b>" + tagEnd;
248                 continue;
249             }
250
251             if (CFStringCompare(key, kCMTextMarkupAttribute_ItalicStyle, 0) == kCFCompareEqualTo) {
252                 if (static_cast<CFBooleanRef>(value) != kCFBooleanTrue)
253                     continue;
254
255                 tagStart.append("<i>");
256                 tagEnd = "</i>" + tagEnd;
257                 continue;
258             }
259
260             if (CFStringCompare(key, kCMTextMarkupAttribute_UnderlineStyle, 0) == kCFCompareEqualTo) {
261                 if (static_cast<CFBooleanRef>(value) != kCFBooleanTrue)
262                     continue;
263
264                 tagStart.append("<u>");
265                 tagEnd = "</u>" + tagEnd;
266                 continue;
267             }
268
269             if (CFStringCompare(key, kCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection, 0) == kCFCompareEqualTo) {
270                 if (CFGetTypeID(value) != CFNumberGetTypeID())
271                     continue;
272                 if (processed & Line)
273                     continue;
274                 processed |= Line;
275
276                 CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
277                 double line;
278                 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &line);
279                 cueData->setLine(line);
280                 continue;
281             }
282
283             if (CFStringCompare(key, kCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection, 0) == kCFCompareEqualTo) {
284                 if (CFGetTypeID(value) != CFNumberGetTypeID())
285                     continue;
286                 if (processed & Position)
287                     continue;
288                 processed |= Position;
289
290                 CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
291                 double position;
292                 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &position);
293                 cueData->setPosition(position);
294                 continue;
295             }
296
297             if (CFStringCompare(key, kCMTextMarkupAttribute_WritingDirectionSizePercentage, 0) == kCFCompareEqualTo) {
298                 if (CFGetTypeID(value) != CFNumberGetTypeID())
299                     continue;
300                 if (processed & Size)
301                     continue;
302                 processed |= Size;
303
304                 CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
305                 double size;
306                 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &size);
307                 cueData->setSize(size);
308                 continue;
309             }
310
311             if (CFStringCompare(key, kCMTextMarkupAttribute_VerticalLayout, 0) == kCFCompareEqualTo) {
312                 valueString = static_cast<CFStringRef>(value);
313                 if (CFGetTypeID(valueString) != CFStringGetTypeID() || !CFStringGetLength(valueString))
314                     continue;
315                 
316                 if (CFStringCompare(valueString, kCMTextVerticalLayout_LeftToRight, 0) == kCFCompareEqualTo)
317                     tagStart.append(leftToRightMark);
318                 else if (CFStringCompare(valueString, kCMTextVerticalLayout_RightToLeft, 0) == kCFCompareEqualTo)
319                     tagStart.append(rightToLeftMark);
320                 continue;
321             }
322
323             if (CFStringCompare(key, kCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight, 0) == kCFCompareEqualTo) {
324                 if (CFGetTypeID(value) != CFNumberGetTypeID())
325                     continue;
326                 
327                 CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
328                 double baseFontSize;
329                 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &baseFontSize);
330                 cueData->setBaseFontSize(baseFontSize);
331                 continue;
332             }
333
334             if (CFStringCompare(key, kCMTextMarkupAttribute_RelativeFontSize, 0) == kCFCompareEqualTo) {
335                 if (CFGetTypeID(value) != CFNumberGetTypeID())
336                     continue;
337                 
338                 CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
339                 double relativeFontSize;
340                 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &relativeFontSize);
341                 cueData->setRelativeFontSize(relativeFontSize);
342                 continue;
343             }
344
345             if (CFStringCompare(key, kCMTextMarkupAttribute_FontFamilyName, 0) == kCFCompareEqualTo) {
346                 valueString = static_cast<CFStringRef>(value);
347                 if (CFGetTypeID(valueString) != CFStringGetTypeID() || !CFStringGetLength(valueString))
348                     continue;
349                 if (processed & FontName)
350                     continue;
351                 processed |= FontName;
352                 
353                 cueData->setFontName(valueString);
354                 continue;
355             }
356
357             if (CFStringCompare(key, kCMTextMarkupAttribute_ForegroundColorARGB, 0) == kCFCompareEqualTo) {
358                 CFArrayRef arrayValue = static_cast<CFArrayRef>(value);
359                 if (CFGetTypeID(arrayValue) != CFArrayGetTypeID())
360                     continue;
361                 
362                 RGBA32 color;
363                 if (!makeRGBA32FromARGBCFArray(arrayValue, color))
364                     continue;
365                 cueData->setForegroundColor(color);
366             }
367             
368             if (CFStringCompare(key, kCMTextMarkupAttribute_BackgroundColorARGB, 0) == kCFCompareEqualTo) {
369                 CFArrayRef arrayValue = static_cast<CFArrayRef>(value);
370                 if (CFGetTypeID(arrayValue) != CFArrayGetTypeID())
371                     continue;
372                 
373                 RGBA32 color;
374                 if (!makeRGBA32FromARGBCFArray(arrayValue, color))
375                     continue;
376                 cueData->setBackgroundColor(color);
377             }
378
379             if (CFStringCompare(key, kCMTextMarkupAttribute_CharacterBackgroundColorARGB, 0) == kCFCompareEqualTo) {
380                 CFArrayRef arrayValue = static_cast<CFArrayRef>(value);
381                 if (CFGetTypeID(arrayValue) != CFArrayGetTypeID())
382                     continue;
383                 
384                 RGBA32 color;
385                 if (!makeRGBA32FromARGBCFArray(arrayValue, color))
386                     continue;
387                 cueData->setHighlightColor(color);
388             }
389         }
390
391         content.append(tagStart);
392         content.append(attributedStringValue.substring(effectiveRange.location, effectiveRange.length));
393         content.append(tagEnd);
394     }
395
396     if (content.length())
397         cueData->setContent(content.toString());
398 }
399
400 void InbandTextTrackPrivateAVF::processCue(CFArrayRef attributedStrings, CFArrayRef nativeSamples, double time)
401 {
402     if (!client())
403         return;
404
405     processAttributedStrings(attributedStrings, time);
406     processNativeSamples(nativeSamples, time);
407 }
408
409 void InbandTextTrackPrivateAVF::processAttributedStrings(CFArrayRef attributedStrings, double time)
410 {
411     LOG(Media, "InbandTextTrackPrivateAVF::processAttributedStrings - %li attributed strings at time %.2f\n", attributedStrings ? CFArrayGetCount(attributedStrings) : 0, time);
412
413     Vector<RefPtr<GenericCueData>> arrivingCues;
414     if (attributedStrings) {
415         CFIndex count = CFArrayGetCount(attributedStrings);
416         for (CFIndex i = 0; i < count; i++) {
417             CFAttributedStringRef attributedString = static_cast<CFAttributedStringRef>(CFArrayGetValueAtIndex(attributedStrings, i));
418             
419             if (!attributedString || !CFAttributedStringGetLength(attributedString))
420                 continue;
421             
422             RefPtr<GenericCueData> cueData = GenericCueData::create();
423             processCueAttributes(attributedString, cueData.get());
424             if (!cueData->content().length())
425                 continue;
426             
427             arrivingCues.append(cueData);
428             
429             cueData->setStartTime(time);
430             cueData->setEndTime(std::numeric_limits<double>::infinity());
431             
432             // AVFoundation cue "position" is to the center of the text so adjust relative to the edge because we will use it to
433             // set CSS "left".
434             if (cueData->position() >= 0 && cueData->size() > 0)
435                 cueData->setPosition(cueData->position() - cueData->size() / 2);
436             
437             LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - considering cue (\"%s\") for time = %.2f, position =  %.2f, line =  %.2f", this, cueData->content().utf8().data(), cueData->startTime(), cueData->position(), cueData->line());
438             
439             cueData->setStatus(GenericCueData::Partial);
440         }
441     }
442
443     if (m_pendingCueStatus != None) {
444         // Cues do not have an explicit duration, they are displayed until the next "cue" (which might be empty) is emitted.
445         m_currentCueEndTime = time;
446
447         if (m_currentCueEndTime >= m_currentCueStartTime) {
448             for (auto& cueData : m_cues) {
449                 // See if one of the newly-arrived cues is an extension of this cue.
450                 Vector<RefPtr<GenericCueData>> nonExtensionCues;
451                 for (auto& arrivingCue : arrivingCues) {
452                     if (!arrivingCue->doesExtendCueData(*cueData))
453                         nonExtensionCues.append(arrivingCue);
454                     else
455                         LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - found an extension cue (\"%s\") for time = %.2f, position =  %.2f, line =  %.2f", this, arrivingCue->content().utf8().data(), arrivingCue->startTime(), arrivingCue->position(), arrivingCue->line());
456                 }
457
458                 bool currentCueIsExtended = (arrivingCues.size() != nonExtensionCues.size());
459
460                 arrivingCues = nonExtensionCues;
461                 
462                 if (currentCueIsExtended)
463                     continue;
464
465                 if (m_pendingCueStatus == Valid) {
466                     cueData->setEndTime(m_currentCueEndTime);
467                     cueData->setStatus(GenericCueData::Complete);
468
469                     LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - updating cue: start=%.2f, end=%.2f, content=\"%s\"", this, cueData->startTime(), m_currentCueEndTime, cueData->content().utf8().data());
470                     client()->updateGenericCue(this, cueData.get());
471                 } else {
472                     // We have to assume that the implicit duration is invalid for cues delivered during a seek because the AVF decode pipeline may not
473                     // see every cue, so DO NOT update cue duration while seeking.
474                     LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - ignoring cue delivered during seek: start=%.2f, end=%.2f, content=\"%s\"", this, cueData->startTime(), m_currentCueEndTime, cueData->content().utf8().data());
475                 }
476             }
477         } else
478             LOG(Media, "InbandTextTrackPrivateAVF::processCue negative length cue(s) ignored: start=%.2f, end=%.2f\n", m_currentCueStartTime, m_currentCueEndTime);
479
480         removeCompletedCues();
481     }
482
483     if (arrivingCues.isEmpty())
484         return;
485
486     m_currentCueStartTime = time;
487
488     for (auto& cueData : arrivingCues) {
489
490         m_cues.append(cueData);
491         
492         LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - adding cue for time = %.2f, position =  %.2f, line =  %.2f", this, cueData->startTime(), cueData->position(), cueData->line());
493
494         client()->addGenericCue(this, cueData.release());
495     }
496
497     m_pendingCueStatus = seeking() ? DeliveredDuringSeek : Valid;
498 }
499
500 void InbandTextTrackPrivateAVF::beginSeeking()
501 {
502     // Forget any partially accumulated cue data as the seek could be to a time outside of the cue's
503     // range, which will mean that the next cue delivered will result in the current cue getting the
504     // incorrect duration.
505     resetCueValues();
506     m_seeking = true;
507 }
508
509 void InbandTextTrackPrivateAVF::disconnect()
510 {
511     m_owner = 0;
512     m_index = 0;
513 }
514
515 void InbandTextTrackPrivateAVF::removeCompletedCues()
516 {
517     if (client()) {
518         long currentCue = m_cues.size() - 1;
519         for (; currentCue > 0; --currentCue) {
520             if (m_cues[currentCue]->status() != GenericCueData::Complete)
521                 continue;
522
523             LOG(Media, "InbandTextTrackPrivateAVF::removeCompletedCues(%p) - removing cue (\"%s\") for time = %.2f, position =  %.2f, line =  %.2f", this, m_cues[currentCue]->content().utf8().data(), m_cues[currentCue]->startTime(), m_cues[currentCue]->position(), m_cues[currentCue]->line());
524             client()->removeGenericCue(this, m_cues[currentCue].get());
525             m_cues.remove(currentCue);
526         }
527     }
528
529     if (m_cues.isEmpty())
530         m_pendingCueStatus = None;
531
532     m_currentCueStartTime = 0;
533     m_currentCueEndTime = 0;
534 }
535
536 void InbandTextTrackPrivateAVF::resetCueValues()
537 {
538     if (m_currentCueEndTime && m_cues.size())
539         LOG(Media, "InbandTextTrackPrivateAVF::resetCueValues flushing data for cues: start=%.2f\n", m_currentCueStartTime);
540
541     if (client()) {
542         for (size_t i = 0; i < m_cues.size(); i++)
543             client()->removeGenericCue(this, m_cues[i].get());
544     }
545
546     m_cues.resize(0);
547     m_pendingCueStatus = None;
548     m_currentCueStartTime = 0;
549     m_currentCueEndTime = 0;
550 }
551
552 void InbandTextTrackPrivateAVF::setMode(InbandTextTrackPrivate::Mode newMode)
553 {
554     if (!m_owner)
555         return;
556
557     InbandTextTrackPrivate::Mode oldMode = mode();
558     InbandTextTrackPrivate::setMode(newMode);
559
560     if (oldMode == newMode)
561         return;
562
563     m_owner->trackModeChanged();
564 }
565
566 void InbandTextTrackPrivateAVF::processNativeSamples(CFArrayRef nativeSamples, double presentationTime)
567 {
568     if (!nativeSamples)
569         return;
570
571     CFIndex count = CFArrayGetCount(nativeSamples);
572     if (!count)
573         return;
574
575     LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples - %li sample buffers at time %.2f\n", count, presentationTime);
576
577     for (CFIndex i = 0; i < count; i++) {
578
579         CMSampleBufferRef sampleBuffer = (CMSampleBufferRef)CFArrayGetValueAtIndex(nativeSamples, i);
580         if (!sampleBuffer)
581             continue;
582
583         CMSampleTimingInfo timingInfo;
584         OSStatus status = CMSampleBufferGetSampleTimingInfo(sampleBuffer, i, &timingInfo);
585         if (status) {
586             LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - CMSampleBufferGetSampleTimingInfo returned error %x for sample %li", this, status, i);
587             continue;
588         }
589
590         CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
591         size_t bufferLength = CMBlockBufferGetDataLength(blockBuffer);
592         if (bufferLength < ISOBox::boxHeaderSize()) {
593             LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - ERROR: CMSampleBuffer size length unexpectedly small (%zu)!!", this, bufferLength);
594             continue;
595         }
596
597         m_sampleInputBuffer.resize(m_sampleInputBuffer.size() + bufferLength);
598         CMBlockBufferCopyDataBytes(blockBuffer, 0, bufferLength, m_sampleInputBuffer.data() + m_sampleInputBuffer.size() - bufferLength);
599
600         RefPtr<ArrayBuffer> buffer = ArrayBuffer::create(m_sampleInputBuffer.data(), m_sampleInputBuffer.size());
601
602         String type = ISOBox::peekType(buffer.get());
603         size_t boxLength = ISOBox::peekLength(buffer.get());
604         if (boxLength > buffer->byteLength()) {
605             LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - ERROR: chunk '%s' size (%zu) larger than buffer length (%u)!!", this, type.utf8().data(), boxLength, buffer->byteLength());
606             continue;
607         }
608
609         LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - chunk type = '%s', size = %zu", this, type.utf8().data(), boxLength);
610
611         if (type == ISOWebVTTCue::boxType()) {
612 #if !PLATFORM(WIN)
613             ISOWebVTTCue cueData = ISOWebVTTCue(MediaTime::createWithDouble(presentationTime), toMediaTime(timingInfo.duration), buffer.get());
614 #else
615             ISOWebVTTCue cueData = ISOWebVTTCue(MediaTime::createWithDouble(presentationTime), MediaTime::createWithDouble(CMTimeGetSeconds(timingInfo.duration)), buffer.get());
616 #endif
617             LOG(Media, "    sample presentation time = %.2f, duration = %.2f", cueData.presentationTime().toDouble(), cueData.duration().toDouble());
618             LOG(Media, "    id = \"%s\", settings = \"%s\", cue text = \"%s\"", cueData.id().utf8().data(), cueData.settings().utf8().data(), cueData.cueText().utf8().data());
619             LOG(Media, "    sourceID = \"%s\", originalStartTime = \"%s\"", cueData.sourceID().utf8().data(), cueData.originalStartTime().utf8().data());
620
621             client()->parseWebVTTCueData(this, cueData);
622         }
623
624         do {
625             if (m_haveReportedVTTHeader)
626                 break;
627
628             CMFormatDescriptionRef formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer);
629             if (!formatDescription)
630                 break;
631
632             CFDictionaryRef extensions = CMFormatDescriptionGetExtensions(formatDescription);
633             if (!extensions)
634                 break;
635
636             CFDictionaryRef sampleDescriptionExtensions = static_cast<CFDictionaryRef>(CFDictionaryGetValue(extensions, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms));
637             if (!sampleDescriptionExtensions)
638                 break;
639             
640             CFDataRef webvttHeaderData = static_cast<CFDataRef>(CFDictionaryGetValue(sampleDescriptionExtensions, CFSTR("vttC")));
641             if (!webvttHeaderData)
642                 break;
643
644             unsigned length = CFDataGetLength(webvttHeaderData);
645             if (!length)
646                 break;
647
648             // A WebVTT header is terminated by "One or more WebVTT line terminators" so append two line feeds to make sure the parser
649             // reccognized this string as a full header.
650             StringBuilder header;
651             header.append(reinterpret_cast<const unsigned char*>(CFDataGetBytePtr(webvttHeaderData)), length);
652             header.append("\n\n");
653
654             LOG(Media, "    vtt header = \n%s", header.toString().utf8().data());
655             client()->parseWebVTTFileHeader(this, header.toString());
656             m_haveReportedVTTHeader = true;
657         } while (0);
658
659         m_sampleInputBuffer.remove(0, boxLength);
660     }
661 }
662
663 } // namespace WebCore
664
665 #endif // ENABLE(VIDEO) && (USE(AVFOUNDATION) || PLATFORM(IOS))