REGRESSION (r224780): Text stroke not applied to video captions.
[WebKit-https.git] / Source / WebCore / page / CaptionUserPreferencesMediaAF.cpp
1 /*
2  * Copyright (C) 2012-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #if ENABLE(VIDEO_TRACK)
29
30 #if !USE(DIRECT2D)
31
32 #include "CaptionUserPreferencesMediaAF.h"
33
34 #include "AudioTrackList.h"
35 #if PLATFORM(WIN)
36 #include <pal/spi/win/CoreTextSPIWin.h>
37 #endif
38 #include "FloatConversion.h"
39 #include "HTMLMediaElement.h"
40 #include "LocalizedStrings.h"
41 #include "Logging.h"
42 #include "MediaControlElements.h"
43 #include "TextTrackList.h"
44 #include "URL.h"
45 #include "UserStyleSheetTypes.h"
46 #include "VTTCue.h"
47 #include <algorithm>
48 #include <wtf/Language.h>
49 #include <wtf/NeverDestroyed.h>
50 #include <wtf/RetainPtr.h>
51 #include <wtf/SoftLinking.h>
52 #include <wtf/text/CString.h>
53 #include <wtf/text/StringBuilder.h>
54
55 #if PLATFORM(IOS)
56 #import "WebCoreThreadRun.h"
57 #endif
58
59 #if COMPILER(MSVC)
60 // See https://msdn.microsoft.com/en-us/library/35bhkfb6.aspx
61 #pragma warning(disable: 4273)
62 #endif
63
64 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
65 #include <CoreText/CoreText.h>
66 #include <MediaAccessibility/MediaAccessibility.h>
67
68 #include "MediaAccessibilitySoftLink.h"
69
70 #if PLATFORM(WIN)
71
72 #ifdef DEBUG_ALL
73 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_DEBUG_LIBRARY(Lib)
74 #else
75 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_LIBRARY(Lib)
76 #endif
77
78 #define SOFT_LINK_AVF(Lib, Name, Type) SOFT_LINK_DLL_IMPORT(Lib, Name, Type)
79 #define SOFT_LINK_AVF_POINTER(Lib, Name, Type) SOFT_LINK_VARIABLE_DLL_IMPORT_OPTIONAL(Lib, Name, Type)
80 #define SOFT_LINK_AVF_FRAMEWORK_IMPORT(Lib, Fun, ReturnType, Arguments, Signature) SOFT_LINK_DLL_IMPORT(Lib, Fun, ReturnType, __cdecl, Arguments, Signature)
81 #define SOFT_LINK_AVF_FRAMEWORK_IMPORT_OPTIONAL(Lib, Fun, ReturnType, Arguments) SOFT_LINK_DLL_IMPORT_OPTIONAL(Lib, Fun, ReturnType, __cdecl, Arguments)
82
83 SOFT_LINK_AVF_FRAMEWORK(CoreMedia)
84 SOFT_LINK_AVF_FRAMEWORK_IMPORT_OPTIONAL(CoreMedia, MTEnableCaption2015Behavior, Boolean, ())
85
86 #else
87
88 SOFT_LINK_FRAMEWORK(MediaToolbox)
89 SOFT_LINK_OPTIONAL(MediaToolbox, MTEnableCaption2015Behavior, Boolean, (), ())
90
91 #endif // PLATFORM(WIN)
92
93 #endif // HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
94
95 namespace WebCore {
96
97 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
98 static void userCaptionPreferencesChangedNotificationCallback(CFNotificationCenterRef, void* observer, CFStringRef, const void *, CFDictionaryRef)
99 {
100 #if !PLATFORM(IOS)
101     static_cast<CaptionUserPreferencesMediaAF*>(observer)->captionPreferencesChanged();
102 #else
103     WebThreadRun(^{
104         static_cast<CaptionUserPreferencesMediaAF*>(observer)->captionPreferencesChanged();
105     });
106 #endif
107 }
108 #endif
109
110 CaptionUserPreferencesMediaAF::CaptionUserPreferencesMediaAF(PageGroup& group)
111     : CaptionUserPreferences(group)
112 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
113     , m_updateStyleSheetTimer(*this, &CaptionUserPreferencesMediaAF::updateTimerFired)
114     , m_listeningForPreferenceChanges(false)
115 #endif
116 {
117     static bool initialized;
118     if (!initialized) {
119         initialized = true;
120
121         MTEnableCaption2015BehaviorPtrType function = MTEnableCaption2015BehaviorPtr();
122         if (!function || !function())
123             return;
124
125         beginBlockingNotifications();
126         CaptionUserPreferences::setCaptionDisplayMode(Manual);
127         setUserPrefersCaptions(false);
128         setUserPrefersSubtitles(false);
129         setUserPrefersTextDescriptions(false);
130         endBlockingNotifications();
131     }
132 }
133
134 CaptionUserPreferencesMediaAF::~CaptionUserPreferencesMediaAF()
135 {
136 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
137     if (kMAXCaptionAppearanceSettingsChangedNotification)
138         CFNotificationCenterRemoveObserver(CFNotificationCenterGetLocalCenter(), this, kMAXCaptionAppearanceSettingsChangedNotification, 0);
139     if (kMAAudibleMediaSettingsChangedNotification)
140         CFNotificationCenterRemoveObserver(CFNotificationCenterGetLocalCenter(), this, kMAAudibleMediaSettingsChangedNotification, 0);
141 #endif
142 }
143
144 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
145
146 CaptionUserPreferences::CaptionDisplayMode CaptionUserPreferencesMediaAF::captionDisplayMode() const
147 {
148     CaptionDisplayMode internalMode = CaptionUserPreferences::captionDisplayMode();
149     if (internalMode == Manual || testingMode() || !MediaAccessibilityLibrary())
150         return internalMode;
151
152     MACaptionAppearanceDisplayType displayType = MACaptionAppearanceGetDisplayType(kMACaptionAppearanceDomainUser);
153     switch (displayType) {
154     case kMACaptionAppearanceDisplayTypeForcedOnly:
155         return ForcedOnly;
156
157     case kMACaptionAppearanceDisplayTypeAutomatic:
158         return Automatic;
159
160     case kMACaptionAppearanceDisplayTypeAlwaysOn:
161         return AlwaysOn;
162     }
163
164     ASSERT_NOT_REACHED();
165     return ForcedOnly;
166 }
167     
168 void CaptionUserPreferencesMediaAF::setCaptionDisplayMode(CaptionUserPreferences::CaptionDisplayMode mode)
169 {
170     if (testingMode() || !MediaAccessibilityLibrary()) {
171         CaptionUserPreferences::setCaptionDisplayMode(mode);
172         return;
173     }
174
175     if (captionDisplayMode() == Manual)
176         return;
177
178     MACaptionAppearanceDisplayType displayType = kMACaptionAppearanceDisplayTypeForcedOnly;
179     switch (mode) {
180     case Automatic:
181         displayType = kMACaptionAppearanceDisplayTypeAutomatic;
182         break;
183     case ForcedOnly:
184         displayType = kMACaptionAppearanceDisplayTypeForcedOnly;
185         break;
186     case AlwaysOn:
187         displayType = kMACaptionAppearanceDisplayTypeAlwaysOn;
188         break;
189     default:
190         ASSERT_NOT_REACHED();
191         break;
192     }
193
194     MACaptionAppearanceSetDisplayType(kMACaptionAppearanceDomainUser, displayType);
195 }
196
197 bool CaptionUserPreferencesMediaAF::userPrefersCaptions() const
198 {
199     bool captionSetting = CaptionUserPreferences::userPrefersCaptions();
200     if (captionSetting || testingMode() || !MediaAccessibilityLibrary())
201         return captionSetting;
202     
203     RetainPtr<CFArrayRef> captioningMediaCharacteristics = adoptCF(MACaptionAppearanceCopyPreferredCaptioningMediaCharacteristics(kMACaptionAppearanceDomainUser));
204     return captioningMediaCharacteristics && CFArrayGetCount(captioningMediaCharacteristics.get());
205 }
206
207 bool CaptionUserPreferencesMediaAF::userPrefersSubtitles() const
208 {
209     bool subtitlesSetting = CaptionUserPreferences::userPrefersSubtitles();
210     if (subtitlesSetting || testingMode() || !MediaAccessibilityLibrary())
211         return subtitlesSetting;
212     
213     RetainPtr<CFArrayRef> captioningMediaCharacteristics = adoptCF(MACaptionAppearanceCopyPreferredCaptioningMediaCharacteristics(kMACaptionAppearanceDomainUser));
214     return !(captioningMediaCharacteristics && CFArrayGetCount(captioningMediaCharacteristics.get()));
215 }
216
217 void CaptionUserPreferencesMediaAF::updateTimerFired()
218 {
219     updateCaptionStyleSheetOverride();
220 }
221
222 void CaptionUserPreferencesMediaAF::setInterestedInCaptionPreferenceChanges()
223 {
224     if (m_listeningForPreferenceChanges)
225         return;
226
227     if (!MediaAccessibilityLibrary())
228         return;
229
230     if (!kMAXCaptionAppearanceSettingsChangedNotification && !canLoad_MediaAccessibility_kMAAudibleMediaSettingsChangedNotification())
231         return;
232
233     m_listeningForPreferenceChanges = true;
234     m_registeringForNotification = true;
235
236     if (kMAXCaptionAppearanceSettingsChangedNotification)
237         CFNotificationCenterAddObserver(CFNotificationCenterGetLocalCenter(), this, userCaptionPreferencesChangedNotificationCallback, kMAXCaptionAppearanceSettingsChangedNotification, 0, CFNotificationSuspensionBehaviorCoalesce);
238     if (canLoad_MediaAccessibility_kMAAudibleMediaSettingsChangedNotification())
239         CFNotificationCenterAddObserver(CFNotificationCenterGetLocalCenter(), this, userCaptionPreferencesChangedNotificationCallback, kMAAudibleMediaSettingsChangedNotification, 0, CFNotificationSuspensionBehaviorCoalesce);
240     m_registeringForNotification = false;
241
242     // Generating and registering the caption stylesheet can be expensive and this method is called indirectly when the parser creates an audio or
243     // video element, so do it after a brief pause.
244     m_updateStyleSheetTimer.startOneShot(0_s);
245 }
246
247 void CaptionUserPreferencesMediaAF::captionPreferencesChanged()
248 {
249     if (m_registeringForNotification)
250         return;
251
252     if (m_listeningForPreferenceChanges)
253         updateCaptionStyleSheetOverride();
254
255     CaptionUserPreferences::captionPreferencesChanged();
256 }
257
258 String CaptionUserPreferencesMediaAF::captionsWindowCSS() const
259 {
260     MACaptionAppearanceBehavior behavior;
261     RetainPtr<CGColorRef> color = adoptCF(MACaptionAppearanceCopyWindowColor(kMACaptionAppearanceDomainUser, &behavior));
262
263     Color windowColor(color.get());
264     if (!windowColor.isValid())
265         windowColor = Color::transparent;
266
267     bool important = behavior == kMACaptionAppearanceBehaviorUseValue;
268     CGFloat opacity = MACaptionAppearanceGetWindowOpacity(kMACaptionAppearanceDomainUser, &behavior);
269     if (!important)
270         important = behavior == kMACaptionAppearanceBehaviorUseValue;
271     String windowStyle = colorPropertyCSS(CSSPropertyBackgroundColor, Color(windowColor.red(), windowColor.green(), windowColor.blue(), static_cast<int>(opacity * 255)), important);
272
273     if (!opacity)
274         return windowStyle;
275
276     return makeString(windowStyle, getPropertyNameString(CSSPropertyPadding), ": .4em !important;");
277 }
278
279 String CaptionUserPreferencesMediaAF::captionsBackgroundCSS() const
280 {
281     // This default value must be the same as the one specified in mediaControls.css for -webkit-media-text-track-past-nodes
282     // and webkit-media-text-track-future-nodes.
283     static NeverDestroyed<Color> defaultBackgroundColor(0, 0, 0, 0.8 * 255);
284
285     MACaptionAppearanceBehavior behavior;
286
287     RetainPtr<CGColorRef> color = adoptCF(MACaptionAppearanceCopyBackgroundColor(kMACaptionAppearanceDomainUser, &behavior));
288     Color backgroundColor(color.get());
289     if (!backgroundColor.isValid())
290         backgroundColor = defaultBackgroundColor;
291
292     bool important = behavior == kMACaptionAppearanceBehaviorUseValue;
293     CGFloat opacity = MACaptionAppearanceGetBackgroundOpacity(kMACaptionAppearanceDomainUser, &behavior);
294     if (!important)
295         important = behavior == kMACaptionAppearanceBehaviorUseValue;
296     return colorPropertyCSS(CSSPropertyBackgroundColor, Color(backgroundColor.red(), backgroundColor.green(), backgroundColor.blue(), static_cast<int>(opacity * 255)), important);
297 }
298
299 Color CaptionUserPreferencesMediaAF::captionsTextColor(bool& important) const
300 {
301     MACaptionAppearanceBehavior behavior;
302     RetainPtr<CGColorRef> color = adoptCF(MACaptionAppearanceCopyForegroundColor(kMACaptionAppearanceDomainUser, &behavior));
303     Color textColor(color.get());
304     if (!textColor.isValid())
305         // This default value must be the same as the one specified in mediaControls.css for -webkit-media-text-track-container.
306         textColor = Color::white;
307     
308     important = behavior == kMACaptionAppearanceBehaviorUseValue;
309     CGFloat opacity = MACaptionAppearanceGetForegroundOpacity(kMACaptionAppearanceDomainUser, &behavior);
310     if (!important)
311         important = behavior == kMACaptionAppearanceBehaviorUseValue;
312     return Color(textColor.red(), textColor.green(), textColor.blue(), static_cast<int>(opacity * 255));
313 }
314     
315 String CaptionUserPreferencesMediaAF::captionsTextColorCSS() const
316 {
317     bool important;
318     Color textColor = captionsTextColor(important);
319
320     if (!textColor.isValid())
321         return emptyString();
322
323     return colorPropertyCSS(CSSPropertyColor, textColor, important);
324 }
325
326 static void appendCSS(StringBuilder& builder, CSSPropertyID id, const String& value, bool important)
327 {
328     builder.append(getPropertyNameString(id));
329     builder.append(':');
330     builder.append(value);
331     if (important)
332         builder.appendLiteral(" !important");
333     builder.append(';');
334 }
335     
336 String CaptionUserPreferencesMediaAF::windowRoundedCornerRadiusCSS() const
337 {
338     MACaptionAppearanceBehavior behavior;
339     CGFloat radius = MACaptionAppearanceGetWindowRoundedCornerRadius(kMACaptionAppearanceDomainUser, &behavior);
340     if (!radius)
341         return emptyString();
342
343     StringBuilder builder;
344     appendCSS(builder, CSSPropertyBorderRadius, String::format("%.02fpx", radius), behavior == kMACaptionAppearanceBehaviorUseValue);
345     return builder.toString();
346 }
347
348 String CaptionUserPreferencesMediaAF::colorPropertyCSS(CSSPropertyID id, const Color& color, bool important) const
349 {
350     StringBuilder builder;
351     appendCSS(builder, id, color.serialized(), important);
352     return builder.toString();
353 }
354
355 bool CaptionUserPreferencesMediaAF::captionStrokeWidthForFont(float fontSize, const String& language, float& strokeWidth, bool& important) const
356 {
357     if (!canLoad_MediaAccessibility_MACaptionAppearanceCopyFontDescriptorWithStrokeForStyle())
358         return false;
359     
360     MACaptionAppearanceBehavior behavior;
361     auto trackLanguage = language.createCFString();
362     CGFloat strokeWidthPt;
363     
364     auto fontDescriptor = adoptCF(MACaptionAppearanceCopyFontDescriptorWithStrokeForStyle(kMACaptionAppearanceDomainUser, &behavior, kMACaptionAppearanceFontStyleDefault, trackLanguage.get(), fontSize, &strokeWidthPt));
365
366     if (!fontDescriptor)
367         return false;
368
369     // Since only half of the stroke is visible because the stroke is drawn before the fill, we double the stroke width here.
370     strokeWidth = strokeWidthPt * 2;
371     important = behavior == kMACaptionAppearanceBehaviorUseValue;
372     
373     return true;
374 }
375
376 String CaptionUserPreferencesMediaAF::captionsTextEdgeCSS() const
377 {
378     static NeverDestroyed<const String> edgeStyleRaised(MAKE_STATIC_STRING_IMPL(" -.1em -.1em .16em "));
379     static NeverDestroyed<const String> edgeStyleDepressed(MAKE_STATIC_STRING_IMPL(" .1em .1em .16em "));
380     static NeverDestroyed<const String> edgeStyleDropShadow(MAKE_STATIC_STRING_IMPL(" 0 .1em .16em "));
381
382     MACaptionAppearanceBehavior behavior;
383     MACaptionAppearanceTextEdgeStyle textEdgeStyle = MACaptionAppearanceGetTextEdgeStyle(kMACaptionAppearanceDomainUser, &behavior);
384     
385     if (textEdgeStyle == kMACaptionAppearanceTextEdgeStyleUndefined || textEdgeStyle == kMACaptionAppearanceTextEdgeStyleNone)
386         return emptyString();
387
388     StringBuilder builder;
389     bool important = behavior == kMACaptionAppearanceBehaviorUseValue;
390     if (textEdgeStyle == kMACaptionAppearanceTextEdgeStyleRaised)
391         appendCSS(builder, CSSPropertyTextShadow, makeString(edgeStyleRaised.get(), " black"), important);
392     else if (textEdgeStyle == kMACaptionAppearanceTextEdgeStyleDepressed)
393         appendCSS(builder, CSSPropertyTextShadow, makeString(edgeStyleDepressed.get(), " black"), important);
394     else if (textEdgeStyle == kMACaptionAppearanceTextEdgeStyleDropShadow)
395         appendCSS(builder, CSSPropertyTextShadow, makeString(edgeStyleDropShadow.get(), " black"), important);
396
397     if (textEdgeStyle == kMACaptionAppearanceTextEdgeStyleDropShadow || textEdgeStyle == kMACaptionAppearanceTextEdgeStyleUniform) {
398         appendCSS(builder, CSSPropertyStrokeColor, "black", important);
399         appendCSS(builder, CSSPropertyPaintOrder, getValueName(CSSValueStroke), important);
400         appendCSS(builder, CSSPropertyStrokeLinejoin, getValueName(CSSValueRound), important);
401         appendCSS(builder, CSSPropertyStrokeLinecap, getValueName(CSSValueRound), important);
402     }
403     
404     return builder.toString();
405 }
406
407 String CaptionUserPreferencesMediaAF::captionsDefaultFontCSS() const
408 {
409     MACaptionAppearanceBehavior behavior;
410     
411     RetainPtr<CTFontDescriptorRef> font = adoptCF(MACaptionAppearanceCopyFontDescriptorForStyle(kMACaptionAppearanceDomainUser, &behavior, kMACaptionAppearanceFontStyleDefault));
412     if (!font)
413         return emptyString();
414
415     RetainPtr<CFTypeRef> name = adoptCF(CTFontDescriptorCopyAttribute(font.get(), kCTFontNameAttribute));
416     if (!name)
417         return emptyString();
418
419     StringBuilder builder;
420     
421     builder.append(getPropertyNameString(CSSPropertyFontFamily));
422     builder.appendLiteral(": \"");
423     builder.append(static_cast<CFStringRef>(name.get()));
424     builder.append('"');
425
426     auto cascadeList = adoptCF(static_cast<CFArrayRef>(CTFontDescriptorCopyAttribute(font.get(), kCTFontCascadeListAttribute)));
427
428     if (cascadeList) {
429         for (CFIndex i = 0; i < CFArrayGetCount(cascadeList.get()); i++) {
430             auto fontCascade = static_cast<CTFontDescriptorRef>(CFArrayGetValueAtIndex(cascadeList.get(), i));
431             if (!fontCascade)
432                 continue;
433             auto fontCascadeName = adoptCF(CTFontDescriptorCopyAttribute(fontCascade, kCTFontNameAttribute));
434             if (!fontCascadeName)
435                 continue;
436             builder.append(", \"");
437             builder.append(static_cast<CFStringRef>(fontCascadeName.get()));
438             builder.append('"');
439         }
440     }
441     
442     if (behavior == kMACaptionAppearanceBehaviorUseValue)
443         builder.appendLiteral(" !important");
444     builder.append(';');
445     
446     return builder.toString();
447 }
448
449 float CaptionUserPreferencesMediaAF::captionFontSizeScaleAndImportance(bool& important) const
450 {
451     if (testingMode() || !MediaAccessibilityLibrary())
452         return CaptionUserPreferences::captionFontSizeScaleAndImportance(important);
453
454     MACaptionAppearanceBehavior behavior;
455     CGFloat characterScale = CaptionUserPreferences::captionFontSizeScaleAndImportance(important);
456     CGFloat scaleAdjustment = MACaptionAppearanceGetRelativeCharacterSize(kMACaptionAppearanceDomainUser, &behavior);
457
458     if (!scaleAdjustment)
459         return characterScale;
460
461     important = behavior == kMACaptionAppearanceBehaviorUseValue;
462 #if defined(__LP64__) && __LP64__
463     return narrowPrecisionToFloat(scaleAdjustment * characterScale);
464 #else
465     return scaleAdjustment * characterScale;
466 #endif
467 }
468
469 void CaptionUserPreferencesMediaAF::setPreferredLanguage(const String& language)
470 {
471     if (CaptionUserPreferences::captionDisplayMode() == Manual)
472         return;
473
474     if (testingMode() || !MediaAccessibilityLibrary()) {
475         CaptionUserPreferences::setPreferredLanguage(language);
476         return;
477     }
478
479     MACaptionAppearanceAddSelectedLanguage(kMACaptionAppearanceDomainUser, language.createCFString().get());
480 }
481
482 Vector<String> CaptionUserPreferencesMediaAF::preferredLanguages() const
483 {
484     if (testingMode() || !MediaAccessibilityLibrary())
485         return CaptionUserPreferences::preferredLanguages();
486
487     Vector<String> platformLanguages = platformUserPreferredLanguages();
488     Vector<String> override = userPreferredLanguagesOverride();
489     if (!override.isEmpty()) {
490         if (platformLanguages.size() != override.size())
491             return override;
492         for (size_t i = 0; i < override.size(); i++) {
493             if (override[i] != platformLanguages[i])
494                 return override;
495         }
496     }
497
498     CFIndex languageCount = 0;
499     RetainPtr<CFArrayRef> languages = adoptCF(MACaptionAppearanceCopySelectedLanguages(kMACaptionAppearanceDomainUser));
500     if (languages)
501         languageCount = CFArrayGetCount(languages.get());
502
503     if (!languageCount)
504         return CaptionUserPreferences::preferredLanguages();
505
506     Vector<String> userPreferredLanguages;
507     userPreferredLanguages.reserveCapacity(languageCount + platformLanguages.size());
508     for (CFIndex i = 0; i < languageCount; i++)
509         userPreferredLanguages.append(static_cast<CFStringRef>(CFArrayGetValueAtIndex(languages.get(), i)));
510
511     userPreferredLanguages.appendVector(platformLanguages);
512
513     return userPreferredLanguages;
514 }
515
516 void CaptionUserPreferencesMediaAF::setPreferredAudioCharacteristic(const String& characteristic)
517 {
518     if (testingMode() || !MediaAccessibilityLibrary())
519         CaptionUserPreferences::setPreferredAudioCharacteristic(characteristic);
520 }
521
522 Vector<String> CaptionUserPreferencesMediaAF::preferredAudioCharacteristics() const
523 {
524     if (testingMode() || !MediaAccessibilityLibrary() || !canLoad_MediaAccessibility_MAAudibleMediaCopyPreferredCharacteristics())
525         return CaptionUserPreferences::preferredAudioCharacteristics();
526
527     CFIndex characteristicCount = 0;
528     RetainPtr<CFArrayRef> characteristics = adoptCF(MAAudibleMediaCopyPreferredCharacteristics());
529     if (characteristics)
530         characteristicCount = CFArrayGetCount(characteristics.get());
531
532     if (!characteristicCount)
533         return CaptionUserPreferences::preferredAudioCharacteristics();
534
535     Vector<String> userPreferredAudioCharacteristics;
536     userPreferredAudioCharacteristics.reserveCapacity(characteristicCount);
537     for (CFIndex i = 0; i < characteristicCount; i++)
538         userPreferredAudioCharacteristics.append(static_cast<CFStringRef>(CFArrayGetValueAtIndex(characteristics.get(), i)));
539
540     return userPreferredAudioCharacteristics;
541 }
542 #endif // HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
543
544 String CaptionUserPreferencesMediaAF::captionsStyleSheetOverride() const
545 {
546     if (testingMode())
547         return CaptionUserPreferences::captionsStyleSheetOverride();
548     
549     StringBuilder captionsOverrideStyleSheet;
550
551 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
552     if (!MediaAccessibilityLibrary())
553         return CaptionUserPreferences::captionsStyleSheetOverride();
554     
555     String captionsColor = captionsTextColorCSS();
556     String edgeStyle = captionsTextEdgeCSS();
557     String fontName = captionsDefaultFontCSS();
558     String background = captionsBackgroundCSS();
559     if (!background.isEmpty() || !captionsColor.isEmpty() || !edgeStyle.isEmpty() || !fontName.isEmpty()) {
560         captionsOverrideStyleSheet.appendLiteral(" video::");
561         captionsOverrideStyleSheet.append(TextTrackCue::cueShadowPseudoId());
562         captionsOverrideStyleSheet.append('{');
563         
564         if (!background.isEmpty())
565             captionsOverrideStyleSheet.append(background);
566         if (!captionsColor.isEmpty())
567             captionsOverrideStyleSheet.append(captionsColor);
568         if (!edgeStyle.isEmpty())
569             captionsOverrideStyleSheet.append(edgeStyle);
570         if (!fontName.isEmpty())
571             captionsOverrideStyleSheet.append(fontName);
572         
573         captionsOverrideStyleSheet.append('}');
574     }
575     
576     String windowColor = captionsWindowCSS();
577     String windowCornerRadius = windowRoundedCornerRadiusCSS();
578     if (!windowColor.isEmpty() || !windowCornerRadius.isEmpty()) {
579         captionsOverrideStyleSheet.appendLiteral(" video::");
580         captionsOverrideStyleSheet.append(VTTCue::cueBackdropShadowPseudoId());
581         captionsOverrideStyleSheet.append('{');
582         
583         if (!windowColor.isEmpty())
584             captionsOverrideStyleSheet.append(windowColor);
585         if (!windowCornerRadius.isEmpty()) {
586             captionsOverrideStyleSheet.append(windowCornerRadius);
587         }
588         
589         captionsOverrideStyleSheet.append('}');
590     }
591 #endif // HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
592
593     LOG(Media, "CaptionUserPreferencesMediaAF::captionsStyleSheetOverrideSetting style to:\n%s", captionsOverrideStyleSheet.toString().utf8().data());
594
595     return captionsOverrideStyleSheet.toString();
596 }
597
598 static String languageIdentifier(const String& languageCode)
599 {
600     if (languageCode.isEmpty())
601         return languageCode;
602
603     String lowercaseLanguageCode = languageCode.convertToASCIILowercase();
604
605     // Need 2U here to disambiguate String::operator[] from operator(NSString*, int)[] in a production build.
606     if (lowercaseLanguageCode.length() >= 3 && (lowercaseLanguageCode[2U] == '_' || lowercaseLanguageCode[2U] == '-'))
607         lowercaseLanguageCode.truncate(2);
608
609     return lowercaseLanguageCode;
610 }
611
612 static void buildDisplayStringForTrackBase(StringBuilder& displayName, const TrackBase& track)
613 {
614     String label = track.label();
615     String trackLanguageIdentifier = track.validBCP47Language();
616
617     RetainPtr<CFLocaleRef> currentLocale = adoptCF(CFLocaleCreate(kCFAllocatorDefault, defaultLanguage().createCFString().get()));
618     RetainPtr<CFStringRef> localeIdentifier = adoptCF(CFLocaleCreateCanonicalLocaleIdentifierFromString(kCFAllocatorDefault, trackLanguageIdentifier.createCFString().get()));
619     RetainPtr<CFStringRef> languageCF = adoptCF(CFLocaleCopyDisplayNameForPropertyValue(currentLocale.get(), kCFLocaleLanguageCode, localeIdentifier.get()));
620     String language = languageCF.get();
621
622     if (!label.isEmpty()) {
623         if (language.isEmpty() || label.contains(language))
624             displayName.append(label);
625         else {
626             RetainPtr<CFDictionaryRef> localeDict = adoptCF(CFLocaleCreateComponentsFromLocaleIdentifier(kCFAllocatorDefault, localeIdentifier.get()));
627             if (localeDict) {
628                 CFStringRef countryCode = 0;
629                 String countryName;
630                 
631                 CFDictionaryGetValueIfPresent(localeDict.get(), kCFLocaleCountryCode, (const void **)&countryCode);
632                 if (countryCode) {
633                     RetainPtr<CFStringRef> countryNameCF = adoptCF(CFLocaleCopyDisplayNameForPropertyValue(currentLocale.get(), kCFLocaleCountryCode, countryCode));
634                     countryName = countryNameCF.get();
635                 }
636                 
637                 if (!countryName.isEmpty())
638                     displayName.append(textTrackCountryAndLanguageMenuItemText(label, countryName, language));
639                 else
640                     displayName.append(textTrackLanguageMenuItemText(label, language));
641             }
642         }
643     } else {
644         String languageAndLocale = adoptCF(CFLocaleCopyDisplayNameForPropertyValue(currentLocale.get(), kCFLocaleIdentifier, trackLanguageIdentifier.createCFString().get())).get();
645         if (!languageAndLocale.isEmpty())
646             displayName.append(languageAndLocale);
647         else if (!language.isEmpty())
648             displayName.append(language);
649         else
650             displayName.append(localeIdentifier.get());
651     }
652 }
653
654 static String trackDisplayName(AudioTrack* track)
655 {
656     StringBuilder displayName;
657     buildDisplayStringForTrackBase(displayName, *track);
658     
659     if (displayName.isEmpty())
660         displayName.append(audioTrackNoLabelText());
661
662     if (track->kind() != AudioTrack::descriptionKeyword())
663         return displayName.toString();
664
665     return audioDescriptionTrackSuffixText(displayName.toString());
666 }
667
668 String CaptionUserPreferencesMediaAF::displayNameForTrack(AudioTrack* track) const
669 {
670     return trackDisplayName(track);
671 }
672
673 static String trackDisplayName(TextTrack* track)
674 {
675     if (track == TextTrack::captionMenuOffItem())
676         return textTrackOffMenuItemText();
677     if (track == TextTrack::captionMenuAutomaticItem())
678         return textTrackAutomaticMenuItemText();
679
680     StringBuilder displayNameBuilder;
681     buildDisplayStringForTrackBase(displayNameBuilder, *track);
682
683     if (displayNameBuilder.isEmpty())
684         displayNameBuilder.append(textTrackNoLabelText());
685
686     String displayName = displayNameBuilder.toString();
687
688     if (track->isClosedCaptions()) {
689         displayName = closedCaptionTrackMenuItemText(displayName);
690         if (track->isEasyToRead())
691             displayName = easyReaderTrackMenuItemText(displayName);
692
693         return displayName;
694     }
695
696     if (track->isSDH())
697         displayName = sdhTrackMenuItemText(displayName);
698
699     if (track->containsOnlyForcedSubtitles())
700         displayName = forcedTrackMenuItemText(displayName);
701
702     if (track->isEasyToRead())
703         displayName = easyReaderTrackMenuItemText(displayName);
704
705     return displayName;
706 }
707
708 String CaptionUserPreferencesMediaAF::displayNameForTrack(TextTrack* track) const
709 {
710     return trackDisplayName(track);
711 }
712
713 int CaptionUserPreferencesMediaAF::textTrackSelectionScore(TextTrack* track, HTMLMediaElement* mediaElement) const
714 {
715     CaptionDisplayMode displayMode = captionDisplayMode();
716     if (displayMode == Manual)
717         return 0;
718
719     bool legacyOverride = mediaElement->webkitClosedCaptionsVisible();
720     if (displayMode == AlwaysOn && (!userPrefersSubtitles() && !userPrefersCaptions() && !legacyOverride))
721         return 0;
722     if (track->kind() != TextTrack::Kind::Captions && track->kind() != TextTrack::Kind::Subtitles && track->kind() != TextTrack::Kind::Forced)
723         return 0;
724     if (!track->isMainProgramContent())
725         return 0;
726
727     bool trackHasOnlyForcedSubtitles = track->containsOnlyForcedSubtitles();
728     if (!legacyOverride && ((trackHasOnlyForcedSubtitles && displayMode != ForcedOnly) || (!trackHasOnlyForcedSubtitles && displayMode == ForcedOnly)))
729         return 0;
730
731     Vector<String> userPreferredCaptionLanguages = preferredLanguages();
732
733     if ((displayMode == Automatic && !legacyOverride) || trackHasOnlyForcedSubtitles) {
734
735         if (!mediaElement || !mediaElement->player())
736             return 0;
737
738         String textTrackLanguage = track->validBCP47Language();
739         if (textTrackLanguage.isEmpty())
740             return 0;
741
742         Vector<String> languageList;
743         languageList.reserveCapacity(1);
744
745         String audioTrackLanguage;
746         if (testingMode())
747             audioTrackLanguage = primaryAudioTrackLanguageOverride();
748         else
749             audioTrackLanguage = mediaElement->player()->languageOfPrimaryAudioTrack();
750
751         if (audioTrackLanguage.isEmpty())
752             return 0;
753
754         bool exactMatch;
755         if (trackHasOnlyForcedSubtitles) {
756             languageList.append(audioTrackLanguage);
757             size_t offset = indexOfBestMatchingLanguageInList(textTrackLanguage, languageList, exactMatch);
758
759             // Only consider a forced-only track if it IS in the same language as the primary audio track.
760             if (offset)
761                 return 0;
762         } else {
763             languageList.append(defaultLanguage());
764
765             // Only enable a text track if the current audio track is NOT in the user's preferred language ...
766             size_t offset = indexOfBestMatchingLanguageInList(audioTrackLanguage, languageList, exactMatch);
767             if (!offset)
768                 return 0;
769
770             // and the text track matches the user's preferred language.
771             offset = indexOfBestMatchingLanguageInList(textTrackLanguage, languageList, exactMatch);
772             if (offset)
773                 return 0;
774         }
775
776         userPreferredCaptionLanguages = languageList;
777     }
778
779     int trackScore = 0;
780
781     if (userPrefersCaptions()) {
782         // When the user prefers accessibility tracks, rank is SDH, then CC, then subtitles.
783         if (track->kind() == TextTrack::Kind::Subtitles)
784             trackScore = 1;
785         else if (track->isClosedCaptions())
786             trackScore = 2;
787         else
788             trackScore = 3;
789     } else {
790         // When the user prefers translation tracks, rank is subtitles, then SDH, then CC tracks.
791         if (track->kind() == TextTrack::Kind::Subtitles)
792             trackScore = 3;
793         else if (!track->isClosedCaptions())
794             trackScore = 2;
795         else
796             trackScore = 1;
797     }
798
799     return trackScore + textTrackLanguageSelectionScore(track, userPreferredCaptionLanguages);
800 }
801
802 static bool textTrackCompare(const RefPtr<TextTrack>& a, const RefPtr<TextTrack>& b)
803 {
804     String preferredLanguageDisplayName = displayNameForLanguageLocale(languageIdentifier(defaultLanguage()));
805     String aLanguageDisplayName = displayNameForLanguageLocale(languageIdentifier(a->validBCP47Language()));
806     String bLanguageDisplayName = displayNameForLanguageLocale(languageIdentifier(b->language()));
807
808     // Tracks in the user's preferred language are always at the top of the menu.
809     bool aIsPreferredLanguage = !codePointCompare(aLanguageDisplayName, preferredLanguageDisplayName);
810     bool bIsPreferredLanguage = !codePointCompare(bLanguageDisplayName, preferredLanguageDisplayName);
811     if ((aIsPreferredLanguage || bIsPreferredLanguage) && (aIsPreferredLanguage != bIsPreferredLanguage))
812         return aIsPreferredLanguage;
813
814     // Tracks not in the user's preferred language sort first by language ...
815     if (codePointCompare(aLanguageDisplayName, bLanguageDisplayName))
816         return codePointCompare(aLanguageDisplayName, bLanguageDisplayName) < 0;
817
818     // ... but when tracks have the same language, main program content sorts next highest ...
819     bool aIsMainContent = a->isMainProgramContent();
820     bool bIsMainContent = b->isMainProgramContent();
821     if ((aIsMainContent || bIsMainContent) && (aIsMainContent != bIsMainContent))
822         return aIsMainContent;
823
824     // ... and main program trakcs sort higher than CC tracks ...
825     bool aIsCC = a->isClosedCaptions();
826     bool bIsCC = b->isClosedCaptions();
827     if ((aIsCC || bIsCC) && (aIsCC != bIsCC)) {
828         if (aIsCC)
829             return aIsMainContent;
830         return bIsMainContent;
831     }
832
833     // ... and tracks of the same type and language sort by the menu item text.
834     return codePointCompare(trackDisplayName(a.get()), trackDisplayName(b.get())) < 0;
835 }
836
837 Vector<RefPtr<AudioTrack>> CaptionUserPreferencesMediaAF::sortedTrackListForMenu(AudioTrackList* trackList)
838 {
839     ASSERT(trackList);
840     
841     Vector<RefPtr<AudioTrack>> tracksForMenu;
842     
843     for (unsigned i = 0, length = trackList->length(); i < length; ++i) {
844         AudioTrack* track = trackList->item(i);
845         String language = displayNameForLanguageLocale(track->validBCP47Language());
846         tracksForMenu.append(track);
847     }
848     
849     std::sort(tracksForMenu.begin(), tracksForMenu.end(), [](auto& a, auto& b) {
850         return codePointCompare(trackDisplayName(a.get()), trackDisplayName(b.get())) < 0;
851     });
852     
853     return tracksForMenu;
854 }
855
856 Vector<RefPtr<TextTrack>> CaptionUserPreferencesMediaAF::sortedTrackListForMenu(TextTrackList* trackList)
857 {
858     ASSERT(trackList);
859
860     Vector<RefPtr<TextTrack>> tracksForMenu;
861     HashSet<String> languagesIncluded;
862     CaptionDisplayMode displayMode = captionDisplayMode();
863     bool prefersAccessibilityTracks = userPrefersCaptions();
864     bool filterTrackList = shouldFilterTrackMenu();
865
866     for (unsigned i = 0, length = trackList->length(); i < length; ++i) {
867         TextTrack* track = trackList->item(i);
868         String language = displayNameForLanguageLocale(track->validBCP47Language());
869
870         if (displayMode == Manual) {
871             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because selection mode is 'manual'", track->kindKeyword().string().utf8().data(), language.utf8().data());
872             tracksForMenu.append(track);
873             continue;
874         }
875
876         auto kind = track->kind();
877         if (kind != TextTrack::Kind::Captions && kind != TextTrack::Kind::Descriptions && kind != TextTrack::Kind::Subtitles)
878             continue;
879
880         if (track->containsOnlyForcedSubtitles()) {
881             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it contains only forced subtitles", track->kindKeyword().string().utf8().data(), language.utf8().data());
882             continue;
883         }
884         
885         if (track->isEasyToRead()) {
886             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because it is 'easy to read'", track->kindKeyword().string().utf8().data(), language.utf8().data());
887             if (!language.isEmpty())
888                 languagesIncluded.add(language);
889             tracksForMenu.append(track);
890             continue;
891         }
892
893         if (track->mode() == TextTrack::Mode::Showing) {
894             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because it is already visible", track->kindKeyword().string().utf8().data(), language.utf8().data());
895             if (!language.isEmpty())
896                 languagesIncluded.add(language);
897             tracksForMenu.append(track);
898             continue;
899         }
900
901         if (!language.isEmpty() && track->isMainProgramContent()) {
902             bool isAccessibilityTrack = track->kind() == TextTrack::Kind::Captions;
903             if (prefersAccessibilityTracks) {
904                 // In the first pass, include only caption tracks if the user prefers accessibility tracks.
905                 if (!isAccessibilityTrack && filterTrackList) {
906                     LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it is NOT an accessibility track", track->kindKeyword().string().utf8().data(), language.utf8().data());
907                     continue;
908                 }
909             } else {
910                 // In the first pass, only include the first non-CC or SDH track with each language if the user prefers translation tracks.
911                 if (isAccessibilityTrack && filterTrackList) {
912                     LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it is an accessibility track", track->kindKeyword().string().utf8().data(), language.utf8().data());
913                     continue;
914                 }
915                 if (languagesIncluded.contains(language) && filterTrackList) {
916                     LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it is not the first with this language", track->kindKeyword().string().utf8().data(), language.utf8().data());
917                     continue;
918                 }
919             }
920         }
921
922         if (!language.isEmpty())
923             languagesIncluded.add(language);
924         tracksForMenu.append(track);
925
926         LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s', is%s main program content", track->kindKeyword().string().utf8().data(), language.utf8().data(), track->isMainProgramContent() ? "" : " NOT");
927     }
928
929     // Now that we have filtered for the user's accessibility/translation preference, add  all tracks with a unique language without regard to track type.
930     for (unsigned i = 0, length = trackList->length(); i < length; ++i) {
931         TextTrack* track = trackList->item(i);
932         String language = displayNameForLanguageLocale(track->language());
933
934         if (tracksForMenu.contains(track))
935             continue;
936
937         auto kind = track->kind();
938         if (kind != TextTrack::Kind::Captions && kind != TextTrack::Kind::Descriptions && kind != TextTrack::Kind::Subtitles)
939             continue;
940
941         // All candidates with no languge were added the first time through.
942         if (language.isEmpty())
943             continue;
944
945         if (track->containsOnlyForcedSubtitles())
946             continue;
947
948         if (!languagesIncluded.contains(language) && track->isMainProgramContent()) {
949             languagesIncluded.add(language);
950             tracksForMenu.append(track);
951             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because it is the only track with this language", track->kindKeyword().string().utf8().data(), language.utf8().data());
952         }
953     }
954
955     if (tracksForMenu.isEmpty())
956         return tracksForMenu;
957
958     std::sort(tracksForMenu.begin(), tracksForMenu.end(), textTrackCompare);
959
960     tracksForMenu.insert(0, TextTrack::captionMenuOffItem());
961     tracksForMenu.insert(1, TextTrack::captionMenuAutomaticItem());
962
963     return tracksForMenu;
964 }
965     
966 }
967
968 #endif
969
970 #endif // ENABLE(VIDEO_TRACK)