98b22bafe2cfbbd6a0c219112350e7c9070fb258
[WebKit-https.git] / Source / WebCore / page / CaptionUserPreferencesMediaAF.cpp
1 /*
2  * Copyright (C) 2012-2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #if ENABLE(VIDEO_TRACK)
29
30 #if !USE(DIRECT2D)
31
32 #include "CaptionUserPreferencesMediaAF.h"
33
34 #include "AudioTrackList.h"
35 #include "FloatConversion.h"
36 #include "HTMLMediaElement.h"
37 #include "URL.h"
38 #include "Language.h"
39 #include "LocalizedStrings.h"
40 #include "Logging.h"
41 #include "MediaControlElements.h"
42 #include "SoftLinking.h"
43 #include "TextTrackList.h"
44 #include "UserStyleSheetTypes.h"
45 #include "VTTCue.h"
46 #include <wtf/NeverDestroyed.h>
47 #include <wtf/PlatformUserPreferredLanguages.h>
48 #include <wtf/RetainPtr.h>
49 #include <wtf/text/CString.h>
50 #include <wtf/text/StringBuilder.h>
51
52 #if PLATFORM(IOS)
53 #import "WebCoreThreadRun.h"
54 #endif
55
56 #if COMPILER(MSVC)
57 // See https://msdn.microsoft.com/en-us/library/35bhkfb6.aspx
58 #pragma warning(disable: 4273)
59 #endif
60
61 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
62 #include <CoreText/CoreText.h>
63 #include <MediaAccessibility/MediaAccessibility.h>
64
65 #include "MediaAccessibilitySoftLink.h"
66
67 #if PLATFORM(WIN)
68
69 #ifdef DEBUG_ALL
70 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_DEBUG_LIBRARY(Lib)
71 #else
72 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_LIBRARY(Lib)
73 #endif
74
75 #define SOFT_LINK_AVF(Lib, Name, Type) SOFT_LINK_DLL_IMPORT(Lib, Name, Type)
76 #define SOFT_LINK_AVF_POINTER(Lib, Name, Type) SOFT_LINK_VARIABLE_DLL_IMPORT_OPTIONAL(Lib, Name, Type)
77 #define SOFT_LINK_AVF_FRAMEWORK_IMPORT(Lib, Fun, ReturnType, Arguments, Signature) SOFT_LINK_DLL_IMPORT(Lib, Fun, ReturnType, __cdecl, Arguments, Signature)
78 #define SOFT_LINK_AVF_FRAMEWORK_IMPORT_OPTIONAL(Lib, Fun, ReturnType, Arguments) SOFT_LINK_DLL_IMPORT_OPTIONAL(Lib, Fun, ReturnType, __cdecl, Arguments)
79
80 // CoreText only needs to be soft-linked on Windows.
81 SOFT_LINK_AVF_FRAMEWORK(CoreText)
82 SOFT_LINK_AVF_FRAMEWORK_IMPORT(CoreText, CTFontDescriptorCopyAttribute,  CFTypeRef, (CTFontDescriptorRef descriptor, CFStringRef attribute), (descriptor, attribute));
83 SOFT_LINK_AVF_POINTER(CoreText, kCTFontNameAttribute, CFStringRef)
84 #define kCTFontNameAttribute getkCTFontNameAttribute()
85
86 #define CTFontDescriptorCopyAttribute softLink_CTFontDescriptorCopyAttribute
87
88 SOFT_LINK_AVF_FRAMEWORK(CoreMedia)
89 SOFT_LINK_AVF_FRAMEWORK_IMPORT_OPTIONAL(CoreMedia, MTEnableCaption2015Behavior, Boolean, ())
90
91 #else
92
93 SOFT_LINK_FRAMEWORK(MediaToolbox)
94 SOFT_LINK_OPTIONAL(MediaToolbox, MTEnableCaption2015Behavior, Boolean, (), ())
95
96 #endif // PLATFORM(WIN)
97
98 #endif // HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
99
100 namespace WebCore {
101
102 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
103 static void userCaptionPreferencesChangedNotificationCallback(CFNotificationCenterRef, void* observer, CFStringRef, const void *, CFDictionaryRef)
104 {
105 #if !PLATFORM(IOS)
106     static_cast<CaptionUserPreferencesMediaAF*>(observer)->captionPreferencesChanged();
107 #else
108     WebThreadRun(^{
109         static_cast<CaptionUserPreferencesMediaAF*>(observer)->captionPreferencesChanged();
110     });
111 #endif
112 }
113 #endif
114
115 CaptionUserPreferencesMediaAF::CaptionUserPreferencesMediaAF(PageGroup& group)
116     : CaptionUserPreferences(group)
117 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
118     , m_updateStyleSheetTimer(*this, &CaptionUserPreferencesMediaAF::updateTimerFired)
119     , m_listeningForPreferenceChanges(false)
120 #endif
121 {
122     static bool initialized;
123     if (!initialized) {
124         initialized = true;
125
126         MTEnableCaption2015BehaviorPtrType function = MTEnableCaption2015BehaviorPtr();
127         if (!function || !function())
128             return;
129
130         beginBlockingNotifications();
131         CaptionUserPreferences::setCaptionDisplayMode(Manual);
132         setUserPrefersCaptions(false);
133         setUserPrefersSubtitles(false);
134         setUserPrefersTextDescriptions(false);
135         endBlockingNotifications();
136     }
137 }
138
139 CaptionUserPreferencesMediaAF::~CaptionUserPreferencesMediaAF()
140 {
141 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
142     if (kMAXCaptionAppearanceSettingsChangedNotification)
143         CFNotificationCenterRemoveObserver(CFNotificationCenterGetLocalCenter(), this, kMAXCaptionAppearanceSettingsChangedNotification, 0);
144     if (kMAAudibleMediaSettingsChangedNotification)
145         CFNotificationCenterRemoveObserver(CFNotificationCenterGetLocalCenter(), this, kMAAudibleMediaSettingsChangedNotification, 0);
146 #endif
147 }
148
149 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
150
151 CaptionUserPreferences::CaptionDisplayMode CaptionUserPreferencesMediaAF::captionDisplayMode() const
152 {
153     CaptionDisplayMode internalMode = CaptionUserPreferences::captionDisplayMode();
154     if (internalMode == Manual || testingMode() || !MediaAccessibilityLibrary())
155         return internalMode;
156
157     MACaptionAppearanceDisplayType displayType = MACaptionAppearanceGetDisplayType(kMACaptionAppearanceDomainUser);
158     switch (displayType) {
159     case kMACaptionAppearanceDisplayTypeForcedOnly:
160         return ForcedOnly;
161
162     case kMACaptionAppearanceDisplayTypeAutomatic:
163         return Automatic;
164
165     case kMACaptionAppearanceDisplayTypeAlwaysOn:
166         return AlwaysOn;
167     }
168
169     ASSERT_NOT_REACHED();
170     return ForcedOnly;
171 }
172     
173 void CaptionUserPreferencesMediaAF::setCaptionDisplayMode(CaptionUserPreferences::CaptionDisplayMode mode)
174 {
175     if (testingMode() || !MediaAccessibilityLibrary()) {
176         CaptionUserPreferences::setCaptionDisplayMode(mode);
177         return;
178     }
179
180     if (captionDisplayMode() == Manual)
181         return;
182
183     MACaptionAppearanceDisplayType displayType = kMACaptionAppearanceDisplayTypeForcedOnly;
184     switch (mode) {
185     case Automatic:
186         displayType = kMACaptionAppearanceDisplayTypeAutomatic;
187         break;
188     case ForcedOnly:
189         displayType = kMACaptionAppearanceDisplayTypeForcedOnly;
190         break;
191     case AlwaysOn:
192         displayType = kMACaptionAppearanceDisplayTypeAlwaysOn;
193         break;
194     default:
195         ASSERT_NOT_REACHED();
196         break;
197     }
198
199     MACaptionAppearanceSetDisplayType(kMACaptionAppearanceDomainUser, displayType);
200 }
201
202 bool CaptionUserPreferencesMediaAF::userPrefersCaptions() const
203 {
204     bool captionSetting = CaptionUserPreferences::userPrefersCaptions();
205     if (captionSetting || testingMode() || !MediaAccessibilityLibrary())
206         return captionSetting;
207     
208     RetainPtr<CFArrayRef> captioningMediaCharacteristics = adoptCF(MACaptionAppearanceCopyPreferredCaptioningMediaCharacteristics(kMACaptionAppearanceDomainUser));
209     return captioningMediaCharacteristics && CFArrayGetCount(captioningMediaCharacteristics.get());
210 }
211
212 bool CaptionUserPreferencesMediaAF::userPrefersSubtitles() const
213 {
214     bool subtitlesSetting = CaptionUserPreferences::userPrefersSubtitles();
215     if (subtitlesSetting || testingMode() || !MediaAccessibilityLibrary())
216         return subtitlesSetting;
217     
218     RetainPtr<CFArrayRef> captioningMediaCharacteristics = adoptCF(MACaptionAppearanceCopyPreferredCaptioningMediaCharacteristics(kMACaptionAppearanceDomainUser));
219     return !(captioningMediaCharacteristics && CFArrayGetCount(captioningMediaCharacteristics.get()));
220 }
221
222 void CaptionUserPreferencesMediaAF::updateTimerFired()
223 {
224     updateCaptionStyleSheetOverride();
225 }
226
227 void CaptionUserPreferencesMediaAF::setInterestedInCaptionPreferenceChanges()
228 {
229     if (m_listeningForPreferenceChanges)
230         return;
231
232     if (!MediaAccessibilityLibrary())
233         return;
234
235     if (!kMAXCaptionAppearanceSettingsChangedNotification && !canLoad_MediaAccessibility_kMAAudibleMediaSettingsChangedNotification())
236         return;
237
238     m_listeningForPreferenceChanges = true;
239     m_registeringForNotification = true;
240
241     if (kMAXCaptionAppearanceSettingsChangedNotification)
242         CFNotificationCenterAddObserver(CFNotificationCenterGetLocalCenter(), this, userCaptionPreferencesChangedNotificationCallback, kMAXCaptionAppearanceSettingsChangedNotification, 0, CFNotificationSuspensionBehaviorCoalesce);
243     if (canLoad_MediaAccessibility_kMAAudibleMediaSettingsChangedNotification())
244         CFNotificationCenterAddObserver(CFNotificationCenterGetLocalCenter(), this, userCaptionPreferencesChangedNotificationCallback, kMAAudibleMediaSettingsChangedNotification, 0, CFNotificationSuspensionBehaviorCoalesce);
245     m_registeringForNotification = false;
246
247     // Generating and registering the caption stylesheet can be expensive and this method is called indirectly when the parser creates an audio or
248     // video element, so do it after a brief pause.
249     m_updateStyleSheetTimer.startOneShot(0);
250 }
251
252 void CaptionUserPreferencesMediaAF::captionPreferencesChanged()
253 {
254     if (m_registeringForNotification)
255         return;
256
257     if (m_listeningForPreferenceChanges)
258         updateCaptionStyleSheetOverride();
259
260     CaptionUserPreferences::captionPreferencesChanged();
261 }
262
263 String CaptionUserPreferencesMediaAF::captionsWindowCSS() const
264 {
265     MACaptionAppearanceBehavior behavior;
266     RetainPtr<CGColorRef> color = adoptCF(MACaptionAppearanceCopyWindowColor(kMACaptionAppearanceDomainUser, &behavior));
267
268     Color windowColor(color.get());
269     if (!windowColor.isValid())
270         windowColor = Color::transparent;
271
272     bool important = behavior == kMACaptionAppearanceBehaviorUseValue;
273     CGFloat opacity = MACaptionAppearanceGetWindowOpacity(kMACaptionAppearanceDomainUser, &behavior);
274     if (!important)
275         important = behavior == kMACaptionAppearanceBehaviorUseValue;
276     String windowStyle = colorPropertyCSS(CSSPropertyBackgroundColor, Color(windowColor.red(), windowColor.green(), windowColor.blue(), static_cast<int>(opacity * 255)), important);
277
278     if (!opacity)
279         return windowStyle;
280
281     return makeString(windowStyle, getPropertyNameString(CSSPropertyPadding), ": .4em !important;");
282 }
283
284 String CaptionUserPreferencesMediaAF::captionsBackgroundCSS() const
285 {
286     // This default value must be the same as the one specified in mediaControls.css for -webkit-media-text-track-past-nodes
287     // and webkit-media-text-track-future-nodes.
288     static NeverDestroyed<Color> defaultBackgroundColor(0, 0, 0, 0.8 * 255);
289
290     MACaptionAppearanceBehavior behavior;
291
292     RetainPtr<CGColorRef> color = adoptCF(MACaptionAppearanceCopyBackgroundColor(kMACaptionAppearanceDomainUser, &behavior));
293     Color backgroundColor(color.get());
294     if (!backgroundColor.isValid())
295         backgroundColor = defaultBackgroundColor;
296
297     bool important = behavior == kMACaptionAppearanceBehaviorUseValue;
298     CGFloat opacity = MACaptionAppearanceGetBackgroundOpacity(kMACaptionAppearanceDomainUser, &behavior);
299     if (!important)
300         important = behavior == kMACaptionAppearanceBehaviorUseValue;
301     return colorPropertyCSS(CSSPropertyBackgroundColor, Color(backgroundColor.red(), backgroundColor.green(), backgroundColor.blue(), static_cast<int>(opacity * 255)), important);
302 }
303
304 Color CaptionUserPreferencesMediaAF::captionsTextColor(bool& important) const
305 {
306     MACaptionAppearanceBehavior behavior;
307     RetainPtr<CGColorRef> color = adoptCF(MACaptionAppearanceCopyForegroundColor(kMACaptionAppearanceDomainUser, &behavior));
308     Color textColor(color.get());
309     if (!textColor.isValid())
310         // This default value must be the same as the one specified in mediaControls.css for -webkit-media-text-track-container.
311         textColor = Color::white;
312     
313     important = behavior == kMACaptionAppearanceBehaviorUseValue;
314     CGFloat opacity = MACaptionAppearanceGetForegroundOpacity(kMACaptionAppearanceDomainUser, &behavior);
315     if (!important)
316         important = behavior == kMACaptionAppearanceBehaviorUseValue;
317     return Color(textColor.red(), textColor.green(), textColor.blue(), static_cast<int>(opacity * 255));
318 }
319     
320 String CaptionUserPreferencesMediaAF::captionsTextColorCSS() const
321 {
322     bool important;
323     Color textColor = captionsTextColor(important);
324
325     if (!textColor.isValid())
326         return emptyString();
327
328     return colorPropertyCSS(CSSPropertyColor, textColor, important);
329 }
330     
331 String CaptionUserPreferencesMediaAF::windowRoundedCornerRadiusCSS() const
332 {
333     MACaptionAppearanceBehavior behavior;
334     CGFloat radius = MACaptionAppearanceGetWindowRoundedCornerRadius(kMACaptionAppearanceDomainUser, &behavior);
335     if (!radius)
336         return emptyString();
337
338     StringBuilder builder;
339     builder.append(getPropertyNameString(CSSPropertyBorderRadius));
340     builder.append(String::format(":%.02fpx", radius));
341     if (behavior == kMACaptionAppearanceBehaviorUseValue)
342         builder.appendLiteral(" !important");
343     builder.append(';');
344
345     return builder.toString();
346 }
347     
348 Color CaptionUserPreferencesMediaAF::captionsEdgeColorForTextColor(const Color& textColor) const
349 {
350     int distanceFromWhite = differenceSquared(textColor, Color::white);
351     int distanceFromBlack = differenceSquared(textColor, Color::black);
352     
353     if (distanceFromWhite < distanceFromBlack)
354         return textColor.dark();
355     
356     return textColor.light();
357 }
358
359 String CaptionUserPreferencesMediaAF::cssPropertyWithTextEdgeColor(CSSPropertyID id, const String& value, const Color& textColor, bool important) const
360 {
361     StringBuilder builder;
362     
363     builder.append(getPropertyNameString(id));
364     builder.append(':');
365     builder.append(value);
366     builder.append(' ');
367     builder.append(captionsEdgeColorForTextColor(textColor).serialized());
368     if (important)
369         builder.appendLiteral(" !important");
370     builder.append(';');
371     
372     return builder.toString();
373 }
374
375 String CaptionUserPreferencesMediaAF::colorPropertyCSS(CSSPropertyID id, const Color& color, bool important) const
376 {
377     StringBuilder builder;
378     
379     builder.append(getPropertyNameString(id));
380     builder.append(':');
381     builder.append(color.serialized());
382     if (important)
383         builder.appendLiteral(" !important");
384     builder.append(';');
385     
386     return builder.toString();
387 }
388
389 String CaptionUserPreferencesMediaAF::captionsTextEdgeCSS() const
390 {
391     static NeverDestroyed<const String> edgeStyleRaised(ASCIILiteral(" -.05em -.05em 0 "));
392     static NeverDestroyed<const String> edgeStyleDepressed(ASCIILiteral(" .05em .05em 0 "));
393     static NeverDestroyed<const String> edgeStyleDropShadow(ASCIILiteral(" .075em .075em 0 "));
394     static NeverDestroyed<const String> edgeStyleUniform(ASCIILiteral(" .03em "));
395
396     bool unused;
397     Color color = captionsTextColor(unused);
398     if (!color.isValid())
399         color = Color { Color::black };
400     color = captionsEdgeColorForTextColor(color);
401
402     MACaptionAppearanceBehavior behavior;
403     MACaptionAppearanceTextEdgeStyle textEdgeStyle = MACaptionAppearanceGetTextEdgeStyle(kMACaptionAppearanceDomainUser, &behavior);
404     switch (textEdgeStyle) {
405     case kMACaptionAppearanceTextEdgeStyleUndefined:
406     case kMACaptionAppearanceTextEdgeStyleNone:
407         return emptyString();
408             
409     case kMACaptionAppearanceTextEdgeStyleRaised:
410         return cssPropertyWithTextEdgeColor(CSSPropertyTextShadow, edgeStyleRaised, color, behavior == kMACaptionAppearanceBehaviorUseValue);
411     case kMACaptionAppearanceTextEdgeStyleDepressed:
412         return cssPropertyWithTextEdgeColor(CSSPropertyTextShadow, edgeStyleDepressed, color, behavior == kMACaptionAppearanceBehaviorUseValue);
413     case kMACaptionAppearanceTextEdgeStyleDropShadow:
414         return cssPropertyWithTextEdgeColor(CSSPropertyTextShadow, edgeStyleDropShadow, color, behavior == kMACaptionAppearanceBehaviorUseValue);
415     case kMACaptionAppearanceTextEdgeStyleUniform:
416         return cssPropertyWithTextEdgeColor(CSSPropertyWebkitTextStroke, edgeStyleUniform, color, behavior == kMACaptionAppearanceBehaviorUseValue);
417             
418     default:
419         ASSERT_NOT_REACHED();
420         break;
421     }
422     
423     return emptyString();
424 }
425
426 String CaptionUserPreferencesMediaAF::captionsDefaultFontCSS() const
427 {
428     MACaptionAppearanceBehavior behavior;
429     
430     RetainPtr<CTFontDescriptorRef> font = adoptCF(MACaptionAppearanceCopyFontDescriptorForStyle(kMACaptionAppearanceDomainUser, &behavior, kMACaptionAppearanceFontStyleDefault));
431     if (!font)
432         return emptyString();
433
434     RetainPtr<CFTypeRef> name = adoptCF(CTFontDescriptorCopyAttribute(font.get(), kCTFontNameAttribute));
435     if (!name)
436         return emptyString();
437     
438     StringBuilder builder;
439     
440     builder.append(getPropertyNameString(CSSPropertyFontFamily));
441     builder.appendLiteral(": \"");
442     builder.append(static_cast<CFStringRef>(name.get()));
443     builder.append('"');
444     if (behavior == kMACaptionAppearanceBehaviorUseValue)
445         builder.appendLiteral(" !important");
446     builder.append(';');
447     
448     return builder.toString();
449 }
450
451 float CaptionUserPreferencesMediaAF::captionFontSizeScaleAndImportance(bool& important) const
452 {
453     if (testingMode() || !MediaAccessibilityLibrary())
454         return CaptionUserPreferences::captionFontSizeScaleAndImportance(important);
455
456     MACaptionAppearanceBehavior behavior;
457     CGFloat characterScale = CaptionUserPreferences::captionFontSizeScaleAndImportance(important);
458     CGFloat scaleAdjustment = MACaptionAppearanceGetRelativeCharacterSize(kMACaptionAppearanceDomainUser, &behavior);
459
460     if (!scaleAdjustment)
461         return characterScale;
462
463     important = behavior == kMACaptionAppearanceBehaviorUseValue;
464 #if defined(__LP64__) && __LP64__
465     return narrowPrecisionToFloat(scaleAdjustment * characterScale);
466 #else
467     return scaleAdjustment * characterScale;
468 #endif
469 }
470
471 void CaptionUserPreferencesMediaAF::setPreferredLanguage(const String& language)
472 {
473     if (CaptionUserPreferences::captionDisplayMode() == Manual)
474         return;
475
476     if (testingMode() || !MediaAccessibilityLibrary()) {
477         CaptionUserPreferences::setPreferredLanguage(language);
478         return;
479     }
480
481     MACaptionAppearanceAddSelectedLanguage(kMACaptionAppearanceDomainUser, language.createCFString().get());
482 }
483
484 Vector<String> CaptionUserPreferencesMediaAF::preferredLanguages() const
485 {
486     if (testingMode() || !MediaAccessibilityLibrary())
487         return CaptionUserPreferences::preferredLanguages();
488
489     Vector<String> platformLanguages = platformUserPreferredLanguages();
490     Vector<String> override = userPreferredLanguagesOverride();
491     if (!override.isEmpty()) {
492         if (platformLanguages.size() != override.size())
493             return override;
494         for (size_t i = 0; i < override.size(); i++) {
495             if (override[i] != platformLanguages[i])
496                 return override;
497         }
498     }
499
500     CFIndex languageCount = 0;
501     RetainPtr<CFArrayRef> languages = adoptCF(MACaptionAppearanceCopySelectedLanguages(kMACaptionAppearanceDomainUser));
502     if (languages)
503         languageCount = CFArrayGetCount(languages.get());
504
505     if (!languageCount)
506         return CaptionUserPreferences::preferredLanguages();
507
508     Vector<String> userPreferredLanguages;
509     userPreferredLanguages.reserveCapacity(languageCount + platformLanguages.size());
510     for (CFIndex i = 0; i < languageCount; i++)
511         userPreferredLanguages.append(static_cast<CFStringRef>(CFArrayGetValueAtIndex(languages.get(), i)));
512
513     userPreferredLanguages.appendVector(platformLanguages);
514
515     return userPreferredLanguages;
516 }
517
518 void CaptionUserPreferencesMediaAF::setPreferredAudioCharacteristic(const String& characteristic)
519 {
520     if (testingMode() || !MediaAccessibilityLibrary())
521         CaptionUserPreferences::setPreferredAudioCharacteristic(characteristic);
522 }
523
524 Vector<String> CaptionUserPreferencesMediaAF::preferredAudioCharacteristics() const
525 {
526     if (testingMode() || !MediaAccessibilityLibrary() || !canLoad_MediaAccessibility_MAAudibleMediaCopyPreferredCharacteristics())
527         return CaptionUserPreferences::preferredAudioCharacteristics();
528
529     CFIndex characteristicCount = 0;
530     RetainPtr<CFArrayRef> characteristics = adoptCF(MAAudibleMediaCopyPreferredCharacteristics());
531     if (characteristics)
532         characteristicCount = CFArrayGetCount(characteristics.get());
533
534     if (!characteristicCount)
535         return CaptionUserPreferences::preferredAudioCharacteristics();
536
537     Vector<String> userPreferredAudioCharacteristics;
538     userPreferredAudioCharacteristics.reserveCapacity(characteristicCount);
539     for (CFIndex i = 0; i < characteristicCount; i++)
540         userPreferredAudioCharacteristics.append(static_cast<CFStringRef>(CFArrayGetValueAtIndex(characteristics.get(), i)));
541
542     return userPreferredAudioCharacteristics;
543 }
544 #endif // HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
545
546 String CaptionUserPreferencesMediaAF::captionsStyleSheetOverride() const
547 {
548     if (testingMode())
549         return CaptionUserPreferences::captionsStyleSheetOverride();
550     
551     StringBuilder captionsOverrideStyleSheet;
552
553 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
554     if (!MediaAccessibilityLibrary())
555         return CaptionUserPreferences::captionsStyleSheetOverride();
556     
557     String captionsColor = captionsTextColorCSS();
558     String edgeStyle = captionsTextEdgeCSS();
559     String fontName = captionsDefaultFontCSS();
560     String background = captionsBackgroundCSS();
561     if (!background.isEmpty() || !captionsColor.isEmpty() || !edgeStyle.isEmpty() || !fontName.isEmpty()) {
562         captionsOverrideStyleSheet.appendLiteral(" video::");
563         captionsOverrideStyleSheet.append(TextTrackCue::cueShadowPseudoId());
564         captionsOverrideStyleSheet.append('{');
565         
566         if (!background.isEmpty())
567             captionsOverrideStyleSheet.append(background);
568         if (!captionsColor.isEmpty())
569             captionsOverrideStyleSheet.append(captionsColor);
570         if (!edgeStyle.isEmpty())
571             captionsOverrideStyleSheet.append(edgeStyle);
572         if (!fontName.isEmpty())
573             captionsOverrideStyleSheet.append(fontName);
574         
575         captionsOverrideStyleSheet.append('}');
576     }
577     
578     String windowColor = captionsWindowCSS();
579     String windowCornerRadius = windowRoundedCornerRadiusCSS();
580     if (!windowColor.isEmpty() || !windowCornerRadius.isEmpty()) {
581         captionsOverrideStyleSheet.appendLiteral(" video::");
582         captionsOverrideStyleSheet.append(VTTCue::cueBackdropShadowPseudoId());
583         captionsOverrideStyleSheet.append('{');
584         
585         if (!windowColor.isEmpty())
586             captionsOverrideStyleSheet.append(windowColor);
587         if (!windowCornerRadius.isEmpty()) {
588             captionsOverrideStyleSheet.append(windowCornerRadius);
589         }
590         
591         captionsOverrideStyleSheet.append('}');
592     }
593 #endif // HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
594
595     LOG(Media, "CaptionUserPreferencesMediaAF::captionsStyleSheetOverrideSetting sytle to:\n%s", captionsOverrideStyleSheet.toString().utf8().data());
596
597     return captionsOverrideStyleSheet.toString();
598 }
599
600 static String languageIdentifier(const String& languageCode)
601 {
602     if (languageCode.isEmpty())
603         return languageCode;
604
605     String lowercaseLanguageCode = languageCode.convertToASCIILowercase();
606
607     // Need 2U here to disambiguate String::operator[] from operator(NSString*, int)[] in a production build.
608     if (lowercaseLanguageCode.length() >= 3 && (lowercaseLanguageCode[2U] == '_' || lowercaseLanguageCode[2U] == '-'))
609         lowercaseLanguageCode.truncate(2);
610
611     return lowercaseLanguageCode;
612 }
613
614 static void buildDisplayStringForTrackBase(StringBuilder& displayName, const TrackBase& track)
615 {
616     String label = track.label();
617     String trackLanguageIdentifier = track.language();
618
619     RetainPtr<CFLocaleRef> currentLocale = adoptCF(CFLocaleCreate(kCFAllocatorDefault, defaultLanguage().createCFString().get()));
620     RetainPtr<CFStringRef> localeIdentifier = adoptCF(CFLocaleCreateCanonicalLocaleIdentifierFromString(kCFAllocatorDefault, trackLanguageIdentifier.createCFString().get()));
621     RetainPtr<CFStringRef> languageCF = adoptCF(CFLocaleCopyDisplayNameForPropertyValue(currentLocale.get(), kCFLocaleLanguageCode, localeIdentifier.get()));
622     String language = languageCF.get();
623
624     if (!label.isEmpty()) {
625         if (language.isEmpty() || label.contains(language))
626             displayName.append(label);
627         else {
628             RetainPtr<CFDictionaryRef> localeDict = adoptCF(CFLocaleCreateComponentsFromLocaleIdentifier(kCFAllocatorDefault, localeIdentifier.get()));
629             if (localeDict) {
630                 CFStringRef countryCode = 0;
631                 String countryName;
632                 
633                 CFDictionaryGetValueIfPresent(localeDict.get(), kCFLocaleCountryCode, (const void **)&countryCode);
634                 if (countryCode) {
635                     RetainPtr<CFStringRef> countryNameCF = adoptCF(CFLocaleCopyDisplayNameForPropertyValue(currentLocale.get(), kCFLocaleCountryCode, countryCode));
636                     countryName = countryNameCF.get();
637                 }
638                 
639                 if (!countryName.isEmpty())
640                     displayName.append(textTrackCountryAndLanguageMenuItemText(label, countryName, language));
641                 else
642                     displayName.append(textTrackLanguageMenuItemText(label, language));
643             }
644         }
645     } else {
646         String languageAndLocale = adoptCF(CFLocaleCopyDisplayNameForPropertyValue(currentLocale.get(), kCFLocaleIdentifier, trackLanguageIdentifier.createCFString().get())).get();
647         if (!languageAndLocale.isEmpty())
648             displayName.append(languageAndLocale);
649         else if (!language.isEmpty())
650             displayName.append(language);
651         else
652             displayName.append(localeIdentifier.get());
653     }
654 }
655
656 static String trackDisplayName(AudioTrack* track)
657 {
658     StringBuilder displayName;
659     buildDisplayStringForTrackBase(displayName, *track);
660     
661     if (displayName.isEmpty())
662         displayName.append(audioTrackNoLabelText());
663
664     if (track->kind() != AudioTrack::descriptionKeyword())
665         return displayName.toString();
666
667     return audioDescriptionTrackSuffixText(displayName.toString());
668 }
669
670 String CaptionUserPreferencesMediaAF::displayNameForTrack(AudioTrack* track) const
671 {
672     return trackDisplayName(track);
673 }
674
675 static String trackDisplayName(TextTrack* track)
676 {
677     if (track == TextTrack::captionMenuOffItem())
678         return textTrackOffMenuItemText();
679     if (track == TextTrack::captionMenuAutomaticItem())
680         return textTrackAutomaticMenuItemText();
681
682     StringBuilder displayNameBuilder;
683     buildDisplayStringForTrackBase(displayNameBuilder, *track);
684
685     if (displayNameBuilder.isEmpty())
686         displayNameBuilder.append(textTrackNoLabelText());
687
688     String displayName = displayNameBuilder.toString();
689
690     if (track->isClosedCaptions()) {
691         displayName = closedCaptionTrackMenuItemText(displayName);
692         if (track->isEasyToRead())
693             displayName = easyReaderTrackMenuItemText(displayName);
694
695         return displayName;
696     }
697
698     if (track->isSDH())
699         displayName = sdhTrackMenuItemText(displayName);
700
701     if (track->containsOnlyForcedSubtitles())
702         displayName = forcedTrackMenuItemText(displayName);
703
704     if (track->isEasyToRead())
705         displayName = easyReaderTrackMenuItemText(displayName);
706
707     return displayName;
708 }
709
710 String CaptionUserPreferencesMediaAF::displayNameForTrack(TextTrack* track) const
711 {
712     return trackDisplayName(track);
713 }
714
715 int CaptionUserPreferencesMediaAF::textTrackSelectionScore(TextTrack* track, HTMLMediaElement* mediaElement) const
716 {
717     CaptionDisplayMode displayMode = captionDisplayMode();
718     if (displayMode == Manual)
719         return 0;
720
721     bool legacyOverride = mediaElement->webkitClosedCaptionsVisible();
722     if (displayMode == AlwaysOn && (!userPrefersSubtitles() && !userPrefersCaptions() && !legacyOverride))
723         return 0;
724     if (track->kind() != TextTrack::Kind::Captions && track->kind() != TextTrack::Kind::Subtitles && track->kind() != TextTrack::Kind::Forced)
725         return 0;
726     if (!track->isMainProgramContent())
727         return 0;
728
729     bool trackHasOnlyForcedSubtitles = track->containsOnlyForcedSubtitles();
730     if (!legacyOverride && ((trackHasOnlyForcedSubtitles && displayMode != ForcedOnly) || (!trackHasOnlyForcedSubtitles && displayMode == ForcedOnly)))
731         return 0;
732
733     Vector<String> userPreferredCaptionLanguages = preferredLanguages();
734
735     if ((displayMode == Automatic && !legacyOverride) || trackHasOnlyForcedSubtitles) {
736
737         if (!mediaElement || !mediaElement->player())
738             return 0;
739
740         String textTrackLanguage = track->language();
741         if (textTrackLanguage.isEmpty())
742             return 0;
743
744         Vector<String> languageList;
745         languageList.reserveCapacity(1);
746
747         String audioTrackLanguage;
748         if (testingMode())
749             audioTrackLanguage = primaryAudioTrackLanguageOverride();
750         else
751             audioTrackLanguage = mediaElement->player()->languageOfPrimaryAudioTrack();
752
753         if (audioTrackLanguage.isEmpty())
754             return 0;
755
756         bool exactMatch;
757         if (trackHasOnlyForcedSubtitles) {
758             languageList.append(audioTrackLanguage);
759             size_t offset = indexOfBestMatchingLanguageInList(textTrackLanguage, languageList, exactMatch);
760
761             // Only consider a forced-only track if it IS in the same language as the primary audio track.
762             if (offset)
763                 return 0;
764         } else {
765             languageList.append(defaultLanguage());
766
767             // Only enable a text track if the current audio track is NOT in the user's preferred language ...
768             size_t offset = indexOfBestMatchingLanguageInList(audioTrackLanguage, languageList, exactMatch);
769             if (!offset)
770                 return 0;
771
772             // and the text track matches the user's preferred language.
773             offset = indexOfBestMatchingLanguageInList(textTrackLanguage, languageList, exactMatch);
774             if (offset)
775                 return 0;
776         }
777
778         userPreferredCaptionLanguages = languageList;
779     }
780
781     int trackScore = 0;
782
783     if (userPrefersCaptions()) {
784         // When the user prefers accessibility tracks, rank is SDH, then CC, then subtitles.
785         if (track->kind() == TextTrack::Kind::Subtitles)
786             trackScore = 1;
787         else if (track->isClosedCaptions())
788             trackScore = 2;
789         else
790             trackScore = 3;
791     } else {
792         // When the user prefers translation tracks, rank is subtitles, then SDH, then CC tracks.
793         if (track->kind() == TextTrack::Kind::Subtitles)
794             trackScore = 3;
795         else if (!track->isClosedCaptions())
796             trackScore = 2;
797         else
798             trackScore = 1;
799     }
800
801     return trackScore + textTrackLanguageSelectionScore(track, userPreferredCaptionLanguages);
802 }
803
804 static bool textTrackCompare(const RefPtr<TextTrack>& a, const RefPtr<TextTrack>& b)
805 {
806     String preferredLanguageDisplayName = displayNameForLanguageLocale(languageIdentifier(defaultLanguage()));
807     String aLanguageDisplayName = displayNameForLanguageLocale(languageIdentifier(a->language()));
808     String bLanguageDisplayName = displayNameForLanguageLocale(languageIdentifier(b->language()));
809
810     // Tracks in the user's preferred language are always at the top of the menu.
811     bool aIsPreferredLanguage = !codePointCompare(aLanguageDisplayName, preferredLanguageDisplayName);
812     bool bIsPreferredLanguage = !codePointCompare(bLanguageDisplayName, preferredLanguageDisplayName);
813     if ((aIsPreferredLanguage || bIsPreferredLanguage) && (aIsPreferredLanguage != bIsPreferredLanguage))
814         return aIsPreferredLanguage;
815
816     // Tracks not in the user's preferred language sort first by language ...
817     if (codePointCompare(aLanguageDisplayName, bLanguageDisplayName))
818         return codePointCompare(aLanguageDisplayName, bLanguageDisplayName) < 0;
819
820     // ... but when tracks have the same language, main program content sorts next highest ...
821     bool aIsMainContent = a->isMainProgramContent();
822     bool bIsMainContent = b->isMainProgramContent();
823     if ((aIsMainContent || bIsMainContent) && (aIsMainContent != bIsMainContent))
824         return aIsMainContent;
825
826     // ... and main program trakcs sort higher than CC tracks ...
827     bool aIsCC = a->isClosedCaptions();
828     bool bIsCC = b->isClosedCaptions();
829     if ((aIsCC || bIsCC) && (aIsCC != bIsCC)) {
830         if (aIsCC)
831             return aIsMainContent;
832         return bIsMainContent;
833     }
834
835     // ... and tracks of the same type and language sort by the menu item text.
836     return codePointCompare(trackDisplayName(a.get()), trackDisplayName(b.get())) < 0;
837 }
838
839 Vector<RefPtr<AudioTrack>> CaptionUserPreferencesMediaAF::sortedTrackListForMenu(AudioTrackList* trackList)
840 {
841     ASSERT(trackList);
842     
843     Vector<RefPtr<AudioTrack>> tracksForMenu;
844     
845     for (unsigned i = 0, length = trackList->length(); i < length; ++i) {
846         AudioTrack* track = trackList->item(i);
847         String language = displayNameForLanguageLocale(track->language());
848         tracksForMenu.append(track);
849     }
850     
851     std::sort(tracksForMenu.begin(), tracksForMenu.end(), [](auto& a, auto& b) {
852         return codePointCompare(trackDisplayName(a.get()), trackDisplayName(b.get())) < 0;
853     });
854     
855     return tracksForMenu;
856 }
857
858 Vector<RefPtr<TextTrack>> CaptionUserPreferencesMediaAF::sortedTrackListForMenu(TextTrackList* trackList)
859 {
860     ASSERT(trackList);
861
862     Vector<RefPtr<TextTrack>> tracksForMenu;
863     HashSet<String> languagesIncluded;
864     CaptionDisplayMode displayMode = captionDisplayMode();
865     bool prefersAccessibilityTracks = userPrefersCaptions();
866     bool filterTrackList = shouldFilterTrackMenu();
867
868     for (unsigned i = 0, length = trackList->length(); i < length; ++i) {
869         TextTrack* track = trackList->item(i);
870         String language = displayNameForLanguageLocale(track->language());
871
872         if (displayMode == Manual) {
873             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because selection mode is 'manual'", track->kindKeyword().string().utf8().data(), language.utf8().data());
874             tracksForMenu.append(track);
875             continue;
876         }
877
878         auto kind = track->kind();
879         if (kind != TextTrack::Kind::Captions && kind != TextTrack::Kind::Descriptions && kind != TextTrack::Kind::Subtitles)
880             continue;
881
882         if (track->containsOnlyForcedSubtitles()) {
883             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it contains only forced subtitles", track->kindKeyword().string().utf8().data(), language.utf8().data());
884             continue;
885         }
886         
887         if (track->isEasyToRead()) {
888             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because it is 'easy to read'", track->kindKeyword().string().utf8().data(), language.utf8().data());
889             if (!language.isEmpty())
890                 languagesIncluded.add(language);
891             tracksForMenu.append(track);
892             continue;
893         }
894
895         if (track->mode() == TextTrack::Mode::Showing) {
896             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because it is already visible", track->kindKeyword().string().utf8().data(), language.utf8().data());
897             if (!language.isEmpty())
898                 languagesIncluded.add(language);
899             tracksForMenu.append(track);
900             continue;
901         }
902
903         if (!language.isEmpty() && track->isMainProgramContent()) {
904             bool isAccessibilityTrack = track->kind() == TextTrack::Kind::Captions;
905             if (prefersAccessibilityTracks) {
906                 // In the first pass, include only caption tracks if the user prefers accessibility tracks.
907                 if (!isAccessibilityTrack && filterTrackList) {
908                     LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it is NOT an accessibility track", track->kindKeyword().string().utf8().data(), language.utf8().data());
909                     continue;
910                 }
911             } else {
912                 // In the first pass, only include the first non-CC or SDH track with each language if the user prefers translation tracks.
913                 if (isAccessibilityTrack && filterTrackList) {
914                     LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it is an accessibility track", track->kindKeyword().string().utf8().data(), language.utf8().data());
915                     continue;
916                 }
917                 if (languagesIncluded.contains(language) && filterTrackList) {
918                     LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it is not the first with this language", track->kindKeyword().string().utf8().data(), language.utf8().data());
919                     continue;
920                 }
921             }
922         }
923
924         if (!language.isEmpty())
925             languagesIncluded.add(language);
926         tracksForMenu.append(track);
927
928         LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s', is%s main program content", track->kindKeyword().string().utf8().data(), language.utf8().data(), track->isMainProgramContent() ? "" : " NOT");
929     }
930
931     // Now that we have filtered for the user's accessibility/translation preference, add  all tracks with a unique language without regard to track type.
932     for (unsigned i = 0, length = trackList->length(); i < length; ++i) {
933         TextTrack* track = trackList->item(i);
934         String language = displayNameForLanguageLocale(track->language());
935
936         if (tracksForMenu.contains(track))
937             continue;
938
939         auto kind = track->kind();
940         if (kind != TextTrack::Kind::Captions && kind != TextTrack::Kind::Descriptions && kind != TextTrack::Kind::Subtitles)
941             continue;
942
943         // All candidates with no languge were added the first time through.
944         if (language.isEmpty())
945             continue;
946
947         if (track->containsOnlyForcedSubtitles())
948             continue;
949
950         if (!languagesIncluded.contains(language) && track->isMainProgramContent()) {
951             languagesIncluded.add(language);
952             tracksForMenu.append(track);
953             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because it is the only track with this language", track->kindKeyword().string().utf8().data(), language.utf8().data());
954         }
955     }
956
957     if (tracksForMenu.isEmpty())
958         return tracksForMenu;
959
960     std::sort(tracksForMenu.begin(), tracksForMenu.end(), textTrackCompare);
961
962     tracksForMenu.insert(0, TextTrack::captionMenuOffItem());
963     tracksForMenu.insert(1, TextTrack::captionMenuAutomaticItem());
964
965     return tracksForMenu;
966 }
967     
968 }
969
970 #endif
971
972 #endif // ENABLE(VIDEO_TRACK)