Unreviewed, rolling out r234489.
[WebKit-https.git] / Source / WebCore / page / CaptionUserPreferencesMediaAF.cpp
1 /*
2  * Copyright (C) 2012-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #if ENABLE(VIDEO_TRACK)
29
30 #if !USE(DIRECT2D)
31
32 #include "CaptionUserPreferencesMediaAF.h"
33
34 #include "AudioTrackList.h"
35 #if PLATFORM(WIN)
36 #include <pal/spi/win/CoreTextSPIWin.h>
37 #endif
38 #include "FloatConversion.h"
39 #include "HTMLMediaElement.h"
40 #include "LocalizedStrings.h"
41 #include "Logging.h"
42 #include "MediaControlElements.h"
43 #include "TextTrackList.h"
44 #include "URL.h"
45 #include "UserStyleSheetTypes.h"
46 #include "VTTCue.h"
47 #include <algorithm>
48 #include <wtf/Language.h>
49 #include <wtf/NeverDestroyed.h>
50 #include <wtf/RetainPtr.h>
51 #include <wtf/SoftLinking.h>
52 #include <wtf/text/CString.h>
53 #include <wtf/text/StringBuilder.h>
54
55 #if PLATFORM(IOS)
56 #import "WebCoreThreadRun.h"
57 #endif
58
59 #if COMPILER(MSVC)
60 // See https://msdn.microsoft.com/en-us/library/35bhkfb6.aspx
61 #pragma warning(disable: 4273)
62 #endif
63
64 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
65 #include <CoreText/CoreText.h>
66 #include <MediaAccessibility/MediaAccessibility.h>
67
68 #include "MediaAccessibilitySoftLink.h"
69
70 #if PLATFORM(WIN)
71
72 #ifdef DEBUG_ALL
73 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_DEBUG_LIBRARY(Lib)
74 #else
75 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_LIBRARY(Lib)
76 #endif
77
78 #define SOFT_LINK_AVF(Lib, Name, Type) SOFT_LINK_DLL_IMPORT(Lib, Name, Type)
79 #define SOFT_LINK_AVF_POINTER(Lib, Name, Type) SOFT_LINK_VARIABLE_DLL_IMPORT_OPTIONAL(Lib, Name, Type)
80 #define SOFT_LINK_AVF_FRAMEWORK_IMPORT(Lib, Fun, ReturnType, Arguments, Signature) SOFT_LINK_DLL_IMPORT(Lib, Fun, ReturnType, __cdecl, Arguments, Signature)
81 #define SOFT_LINK_AVF_FRAMEWORK_IMPORT_OPTIONAL(Lib, Fun, ReturnType, Arguments) SOFT_LINK_DLL_IMPORT_OPTIONAL(Lib, Fun, ReturnType, __cdecl, Arguments)
82
83 SOFT_LINK_AVF_FRAMEWORK(CoreMedia)
84 SOFT_LINK_AVF_FRAMEWORK_IMPORT_OPTIONAL(CoreMedia, MTEnableCaption2015Behavior, Boolean, ())
85
86 #else
87
88 SOFT_LINK_FRAMEWORK_OPTIONAL(MediaToolbox)
89 SOFT_LINK_OPTIONAL(MediaToolbox, MTEnableCaption2015Behavior, Boolean, (), ())
90
91 #endif // PLATFORM(WIN)
92
93 #endif // HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
94
95 namespace WebCore {
96
97 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
98 static void userCaptionPreferencesChangedNotificationCallback(CFNotificationCenterRef, void* observer, CFStringRef, const void *, CFDictionaryRef)
99 {
100 #if !PLATFORM(IOS)
101     static_cast<CaptionUserPreferencesMediaAF*>(observer)->captionPreferencesChanged();
102 #else
103     WebThreadRun(^{
104         static_cast<CaptionUserPreferencesMediaAF*>(observer)->captionPreferencesChanged();
105     });
106 #endif
107 }
108 #endif
109
110 CaptionUserPreferencesMediaAF::CaptionUserPreferencesMediaAF(PageGroup& group)
111     : CaptionUserPreferences(group)
112 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
113     , m_updateStyleSheetTimer(*this, &CaptionUserPreferencesMediaAF::updateTimerFired)
114     , m_listeningForPreferenceChanges(false)
115 #endif
116 {
117     static bool initialized;
118     if (!initialized) {
119         initialized = true;
120
121 #if !PLATFORM(WIN)
122         if (!MediaToolboxLibrary())
123             return;
124 #endif
125
126         MTEnableCaption2015BehaviorPtrType function = MTEnableCaption2015BehaviorPtr();
127         if (!function || !function())
128             return;
129
130         beginBlockingNotifications();
131         CaptionUserPreferences::setCaptionDisplayMode(Manual);
132         setUserPrefersCaptions(false);
133         setUserPrefersSubtitles(false);
134         setUserPrefersTextDescriptions(false);
135         endBlockingNotifications();
136     }
137 }
138
139 CaptionUserPreferencesMediaAF::~CaptionUserPreferencesMediaAF()
140 {
141 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
142     if (kMAXCaptionAppearanceSettingsChangedNotification)
143         CFNotificationCenterRemoveObserver(CFNotificationCenterGetLocalCenter(), this, kMAXCaptionAppearanceSettingsChangedNotification, 0);
144     if (kMAAudibleMediaSettingsChangedNotification)
145         CFNotificationCenterRemoveObserver(CFNotificationCenterGetLocalCenter(), this, kMAAudibleMediaSettingsChangedNotification, 0);
146 #endif
147 }
148
149 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
150
151 CaptionUserPreferences::CaptionDisplayMode CaptionUserPreferencesMediaAF::captionDisplayMode() const
152 {
153     CaptionDisplayMode internalMode = CaptionUserPreferences::captionDisplayMode();
154     if (internalMode == Manual || testingMode() || !MediaAccessibilityLibrary())
155         return internalMode;
156
157     MACaptionAppearanceDisplayType displayType = MACaptionAppearanceGetDisplayType(kMACaptionAppearanceDomainUser);
158     switch (displayType) {
159     case kMACaptionAppearanceDisplayTypeForcedOnly:
160         return ForcedOnly;
161
162     case kMACaptionAppearanceDisplayTypeAutomatic:
163         return Automatic;
164
165     case kMACaptionAppearanceDisplayTypeAlwaysOn:
166         return AlwaysOn;
167     }
168
169     ASSERT_NOT_REACHED();
170     return ForcedOnly;
171 }
172     
173 void CaptionUserPreferencesMediaAF::setCaptionDisplayMode(CaptionUserPreferences::CaptionDisplayMode mode)
174 {
175     if (testingMode() || !MediaAccessibilityLibrary()) {
176         CaptionUserPreferences::setCaptionDisplayMode(mode);
177         return;
178     }
179
180     if (captionDisplayMode() == Manual)
181         return;
182
183     MACaptionAppearanceDisplayType displayType = kMACaptionAppearanceDisplayTypeForcedOnly;
184     switch (mode) {
185     case Automatic:
186         displayType = kMACaptionAppearanceDisplayTypeAutomatic;
187         break;
188     case ForcedOnly:
189         displayType = kMACaptionAppearanceDisplayTypeForcedOnly;
190         break;
191     case AlwaysOn:
192         displayType = kMACaptionAppearanceDisplayTypeAlwaysOn;
193         break;
194     default:
195         ASSERT_NOT_REACHED();
196         break;
197     }
198
199     MACaptionAppearanceSetDisplayType(kMACaptionAppearanceDomainUser, displayType);
200 }
201
202 bool CaptionUserPreferencesMediaAF::userPrefersCaptions() const
203 {
204     bool captionSetting = CaptionUserPreferences::userPrefersCaptions();
205     if (captionSetting || testingMode() || !MediaAccessibilityLibrary())
206         return captionSetting;
207     
208     RetainPtr<CFArrayRef> captioningMediaCharacteristics = adoptCF(MACaptionAppearanceCopyPreferredCaptioningMediaCharacteristics(kMACaptionAppearanceDomainUser));
209     return captioningMediaCharacteristics && CFArrayGetCount(captioningMediaCharacteristics.get());
210 }
211
212 bool CaptionUserPreferencesMediaAF::userPrefersSubtitles() const
213 {
214     bool subtitlesSetting = CaptionUserPreferences::userPrefersSubtitles();
215     if (subtitlesSetting || testingMode() || !MediaAccessibilityLibrary())
216         return subtitlesSetting;
217     
218     RetainPtr<CFArrayRef> captioningMediaCharacteristics = adoptCF(MACaptionAppearanceCopyPreferredCaptioningMediaCharacteristics(kMACaptionAppearanceDomainUser));
219     return !(captioningMediaCharacteristics && CFArrayGetCount(captioningMediaCharacteristics.get()));
220 }
221
222 void CaptionUserPreferencesMediaAF::updateTimerFired()
223 {
224     updateCaptionStyleSheetOverride();
225 }
226
227 void CaptionUserPreferencesMediaAF::setInterestedInCaptionPreferenceChanges()
228 {
229     if (m_listeningForPreferenceChanges)
230         return;
231
232     if (!MediaAccessibilityLibrary())
233         return;
234
235     if (!kMAXCaptionAppearanceSettingsChangedNotification && !canLoad_MediaAccessibility_kMAAudibleMediaSettingsChangedNotification())
236         return;
237
238     m_listeningForPreferenceChanges = true;
239     m_registeringForNotification = true;
240
241     if (kMAXCaptionAppearanceSettingsChangedNotification)
242         CFNotificationCenterAddObserver(CFNotificationCenterGetLocalCenter(), this, userCaptionPreferencesChangedNotificationCallback, kMAXCaptionAppearanceSettingsChangedNotification, 0, CFNotificationSuspensionBehaviorCoalesce);
243     if (canLoad_MediaAccessibility_kMAAudibleMediaSettingsChangedNotification())
244         CFNotificationCenterAddObserver(CFNotificationCenterGetLocalCenter(), this, userCaptionPreferencesChangedNotificationCallback, kMAAudibleMediaSettingsChangedNotification, 0, CFNotificationSuspensionBehaviorCoalesce);
245     m_registeringForNotification = false;
246
247     // Generating and registering the caption stylesheet can be expensive and this method is called indirectly when the parser creates an audio or
248     // video element, so do it after a brief pause.
249     m_updateStyleSheetTimer.startOneShot(0_s);
250 }
251
252 void CaptionUserPreferencesMediaAF::captionPreferencesChanged()
253 {
254     if (m_registeringForNotification)
255         return;
256
257     if (m_listeningForPreferenceChanges)
258         updateCaptionStyleSheetOverride();
259
260     CaptionUserPreferences::captionPreferencesChanged();
261 }
262
263 String CaptionUserPreferencesMediaAF::captionsWindowCSS() const
264 {
265     MACaptionAppearanceBehavior behavior;
266     RetainPtr<CGColorRef> color = adoptCF(MACaptionAppearanceCopyWindowColor(kMACaptionAppearanceDomainUser, &behavior));
267
268     Color windowColor(color.get());
269     if (!windowColor.isValid())
270         windowColor = Color::transparent;
271
272     bool important = behavior == kMACaptionAppearanceBehaviorUseValue;
273     CGFloat opacity = MACaptionAppearanceGetWindowOpacity(kMACaptionAppearanceDomainUser, &behavior);
274     if (!important)
275         important = behavior == kMACaptionAppearanceBehaviorUseValue;
276     String windowStyle = colorPropertyCSS(CSSPropertyBackgroundColor, Color(windowColor.red(), windowColor.green(), windowColor.blue(), static_cast<int>(opacity * 255)), important);
277
278     if (!opacity)
279         return windowStyle;
280
281     return makeString(windowStyle, getPropertyNameString(CSSPropertyPadding), ": .4em !important;");
282 }
283
284 String CaptionUserPreferencesMediaAF::captionsBackgroundCSS() const
285 {
286     // This default value must be the same as the one specified in mediaControls.css for -webkit-media-text-track-past-nodes
287     // and webkit-media-text-track-future-nodes.
288     static NeverDestroyed<Color> defaultBackgroundColor(0, 0, 0, 0.8 * 255);
289
290     MACaptionAppearanceBehavior behavior;
291
292     RetainPtr<CGColorRef> color = adoptCF(MACaptionAppearanceCopyBackgroundColor(kMACaptionAppearanceDomainUser, &behavior));
293     Color backgroundColor(color.get());
294     if (!backgroundColor.isValid())
295         backgroundColor = defaultBackgroundColor;
296
297     bool important = behavior == kMACaptionAppearanceBehaviorUseValue;
298     CGFloat opacity = MACaptionAppearanceGetBackgroundOpacity(kMACaptionAppearanceDomainUser, &behavior);
299     if (!important)
300         important = behavior == kMACaptionAppearanceBehaviorUseValue;
301     return colorPropertyCSS(CSSPropertyBackgroundColor, Color(backgroundColor.red(), backgroundColor.green(), backgroundColor.blue(), static_cast<int>(opacity * 255)), important);
302 }
303
304 Color CaptionUserPreferencesMediaAF::captionsTextColor(bool& important) const
305 {
306     MACaptionAppearanceBehavior behavior;
307     RetainPtr<CGColorRef> color = adoptCF(MACaptionAppearanceCopyForegroundColor(kMACaptionAppearanceDomainUser, &behavior));
308     Color textColor(color.get());
309     if (!textColor.isValid())
310         // This default value must be the same as the one specified in mediaControls.css for -webkit-media-text-track-container.
311         textColor = Color::white;
312     
313     important = behavior == kMACaptionAppearanceBehaviorUseValue;
314     CGFloat opacity = MACaptionAppearanceGetForegroundOpacity(kMACaptionAppearanceDomainUser, &behavior);
315     if (!important)
316         important = behavior == kMACaptionAppearanceBehaviorUseValue;
317     return Color(textColor.red(), textColor.green(), textColor.blue(), static_cast<int>(opacity * 255));
318 }
319     
320 String CaptionUserPreferencesMediaAF::captionsTextColorCSS() const
321 {
322     bool important;
323     Color textColor = captionsTextColor(important);
324
325     if (!textColor.isValid())
326         return emptyString();
327
328     return colorPropertyCSS(CSSPropertyColor, textColor, important);
329 }
330
331 static void appendCSS(StringBuilder& builder, CSSPropertyID id, const String& value, bool important)
332 {
333     builder.append(getPropertyNameString(id));
334     builder.append(':');
335     builder.append(value);
336     if (important)
337         builder.appendLiteral(" !important");
338     builder.append(';');
339 }
340     
341 String CaptionUserPreferencesMediaAF::windowRoundedCornerRadiusCSS() const
342 {
343     MACaptionAppearanceBehavior behavior;
344     CGFloat radius = MACaptionAppearanceGetWindowRoundedCornerRadius(kMACaptionAppearanceDomainUser, &behavior);
345     if (!radius)
346         return emptyString();
347
348     StringBuilder builder;
349     appendCSS(builder, CSSPropertyBorderRadius, String::format("%.02fpx", radius), behavior == kMACaptionAppearanceBehaviorUseValue);
350     return builder.toString();
351 }
352
353 String CaptionUserPreferencesMediaAF::colorPropertyCSS(CSSPropertyID id, const Color& color, bool important) const
354 {
355     StringBuilder builder;
356     appendCSS(builder, id, color.serialized(), important);
357     return builder.toString();
358 }
359
360 bool CaptionUserPreferencesMediaAF::captionStrokeWidthForFont(float fontSize, const String& language, float& strokeWidth, bool& important) const
361 {
362     if (!canLoad_MediaAccessibility_MACaptionAppearanceCopyFontDescriptorWithStrokeForStyle())
363         return false;
364     
365     MACaptionAppearanceBehavior behavior;
366     auto trackLanguage = language.createCFString();
367     CGFloat strokeWidthPt;
368     
369     auto fontDescriptor = adoptCF(MACaptionAppearanceCopyFontDescriptorWithStrokeForStyle(kMACaptionAppearanceDomainUser, &behavior, kMACaptionAppearanceFontStyleDefault, trackLanguage.get(), fontSize, &strokeWidthPt));
370
371     if (!fontDescriptor)
372         return false;
373
374     // Since only half of the stroke is visible because the stroke is drawn before the fill, we double the stroke width here.
375     strokeWidth = strokeWidthPt * 2;
376     important = behavior == kMACaptionAppearanceBehaviorUseValue;
377     
378     return true;
379 }
380
381 String CaptionUserPreferencesMediaAF::captionsTextEdgeCSS() const
382 {
383     static NeverDestroyed<const String> edgeStyleRaised(MAKE_STATIC_STRING_IMPL(" -.1em -.1em .16em "));
384     static NeverDestroyed<const String> edgeStyleDepressed(MAKE_STATIC_STRING_IMPL(" .1em .1em .16em "));
385     static NeverDestroyed<const String> edgeStyleDropShadow(MAKE_STATIC_STRING_IMPL(" 0 .1em .16em "));
386
387     MACaptionAppearanceBehavior behavior;
388     MACaptionAppearanceTextEdgeStyle textEdgeStyle = MACaptionAppearanceGetTextEdgeStyle(kMACaptionAppearanceDomainUser, &behavior);
389     
390     if (textEdgeStyle == kMACaptionAppearanceTextEdgeStyleUndefined || textEdgeStyle == kMACaptionAppearanceTextEdgeStyleNone)
391         return emptyString();
392
393     StringBuilder builder;
394     bool important = behavior == kMACaptionAppearanceBehaviorUseValue;
395     if (textEdgeStyle == kMACaptionAppearanceTextEdgeStyleRaised)
396         appendCSS(builder, CSSPropertyTextShadow, makeString(edgeStyleRaised.get(), " black"), important);
397     else if (textEdgeStyle == kMACaptionAppearanceTextEdgeStyleDepressed)
398         appendCSS(builder, CSSPropertyTextShadow, makeString(edgeStyleDepressed.get(), " black"), important);
399     else if (textEdgeStyle == kMACaptionAppearanceTextEdgeStyleDropShadow)
400         appendCSS(builder, CSSPropertyTextShadow, makeString(edgeStyleDropShadow.get(), " black"), important);
401
402     if (textEdgeStyle == kMACaptionAppearanceTextEdgeStyleDropShadow || textEdgeStyle == kMACaptionAppearanceTextEdgeStyleUniform) {
403         appendCSS(builder, CSSPropertyStrokeColor, "black", important);
404         appendCSS(builder, CSSPropertyPaintOrder, getValueName(CSSValueStroke), important);
405         appendCSS(builder, CSSPropertyStrokeLinejoin, getValueName(CSSValueRound), important);
406         appendCSS(builder, CSSPropertyStrokeLinecap, getValueName(CSSValueRound), important);
407     }
408     
409     return builder.toString();
410 }
411
412 String CaptionUserPreferencesMediaAF::captionsDefaultFontCSS() const
413 {
414     MACaptionAppearanceBehavior behavior;
415     
416     RetainPtr<CTFontDescriptorRef> font = adoptCF(MACaptionAppearanceCopyFontDescriptorForStyle(kMACaptionAppearanceDomainUser, &behavior, kMACaptionAppearanceFontStyleDefault));
417     if (!font)
418         return emptyString();
419
420     RetainPtr<CFTypeRef> name = adoptCF(CTFontDescriptorCopyAttribute(font.get(), kCTFontNameAttribute));
421     if (!name)
422         return emptyString();
423
424     StringBuilder builder;
425     
426     builder.append(getPropertyNameString(CSSPropertyFontFamily));
427     builder.appendLiteral(": \"");
428     builder.append(static_cast<CFStringRef>(name.get()));
429     builder.append('"');
430
431     auto cascadeList = adoptCF(static_cast<CFArrayRef>(CTFontDescriptorCopyAttribute(font.get(), kCTFontCascadeListAttribute)));
432
433     if (cascadeList) {
434         for (CFIndex i = 0; i < CFArrayGetCount(cascadeList.get()); i++) {
435             auto fontCascade = static_cast<CTFontDescriptorRef>(CFArrayGetValueAtIndex(cascadeList.get(), i));
436             if (!fontCascade)
437                 continue;
438             auto fontCascadeName = adoptCF(CTFontDescriptorCopyAttribute(fontCascade, kCTFontNameAttribute));
439             if (!fontCascadeName)
440                 continue;
441             builder.append(", \"");
442             builder.append(static_cast<CFStringRef>(fontCascadeName.get()));
443             builder.append('"');
444         }
445     }
446     
447     if (behavior == kMACaptionAppearanceBehaviorUseValue)
448         builder.appendLiteral(" !important");
449     builder.append(';');
450     
451     return builder.toString();
452 }
453
454 float CaptionUserPreferencesMediaAF::captionFontSizeScaleAndImportance(bool& important) const
455 {
456     if (testingMode() || !MediaAccessibilityLibrary())
457         return CaptionUserPreferences::captionFontSizeScaleAndImportance(important);
458
459     MACaptionAppearanceBehavior behavior;
460     CGFloat characterScale = CaptionUserPreferences::captionFontSizeScaleAndImportance(important);
461     CGFloat scaleAdjustment = MACaptionAppearanceGetRelativeCharacterSize(kMACaptionAppearanceDomainUser, &behavior);
462
463     if (!scaleAdjustment)
464         return characterScale;
465
466     important = behavior == kMACaptionAppearanceBehaviorUseValue;
467 #if defined(__LP64__) && __LP64__
468     return narrowPrecisionToFloat(scaleAdjustment * characterScale);
469 #else
470     return scaleAdjustment * characterScale;
471 #endif
472 }
473
474 void CaptionUserPreferencesMediaAF::setPreferredLanguage(const String& language)
475 {
476     if (CaptionUserPreferences::captionDisplayMode() == Manual)
477         return;
478
479     if (testingMode() || !MediaAccessibilityLibrary()) {
480         CaptionUserPreferences::setPreferredLanguage(language);
481         return;
482     }
483
484     MACaptionAppearanceAddSelectedLanguage(kMACaptionAppearanceDomainUser, language.createCFString().get());
485 }
486
487 Vector<String> CaptionUserPreferencesMediaAF::preferredLanguages() const
488 {
489     if (testingMode() || !MediaAccessibilityLibrary())
490         return CaptionUserPreferences::preferredLanguages();
491
492     Vector<String> platformLanguages = platformUserPreferredLanguages();
493     Vector<String> override = userPreferredLanguagesOverride();
494     if (!override.isEmpty()) {
495         if (platformLanguages.size() != override.size())
496             return override;
497         for (size_t i = 0; i < override.size(); i++) {
498             if (override[i] != platformLanguages[i])
499                 return override;
500         }
501     }
502
503     CFIndex languageCount = 0;
504     RetainPtr<CFArrayRef> languages = adoptCF(MACaptionAppearanceCopySelectedLanguages(kMACaptionAppearanceDomainUser));
505     if (languages)
506         languageCount = CFArrayGetCount(languages.get());
507
508     if (!languageCount)
509         return CaptionUserPreferences::preferredLanguages();
510
511     Vector<String> userPreferredLanguages;
512     userPreferredLanguages.reserveCapacity(languageCount + platformLanguages.size());
513     for (CFIndex i = 0; i < languageCount; i++)
514         userPreferredLanguages.append(static_cast<CFStringRef>(CFArrayGetValueAtIndex(languages.get(), i)));
515
516     userPreferredLanguages.appendVector(platformLanguages);
517
518     return userPreferredLanguages;
519 }
520
521 void CaptionUserPreferencesMediaAF::setPreferredAudioCharacteristic(const String& characteristic)
522 {
523     if (testingMode() || !MediaAccessibilityLibrary())
524         CaptionUserPreferences::setPreferredAudioCharacteristic(characteristic);
525 }
526
527 Vector<String> CaptionUserPreferencesMediaAF::preferredAudioCharacteristics() const
528 {
529     if (testingMode() || !MediaAccessibilityLibrary() || !canLoad_MediaAccessibility_MAAudibleMediaCopyPreferredCharacteristics())
530         return CaptionUserPreferences::preferredAudioCharacteristics();
531
532     CFIndex characteristicCount = 0;
533     RetainPtr<CFArrayRef> characteristics = adoptCF(MAAudibleMediaCopyPreferredCharacteristics());
534     if (characteristics)
535         characteristicCount = CFArrayGetCount(characteristics.get());
536
537     if (!characteristicCount)
538         return CaptionUserPreferences::preferredAudioCharacteristics();
539
540     Vector<String> userPreferredAudioCharacteristics;
541     userPreferredAudioCharacteristics.reserveCapacity(characteristicCount);
542     for (CFIndex i = 0; i < characteristicCount; i++)
543         userPreferredAudioCharacteristics.append(static_cast<CFStringRef>(CFArrayGetValueAtIndex(characteristics.get(), i)));
544
545     return userPreferredAudioCharacteristics;
546 }
547 #endif // HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
548
549 String CaptionUserPreferencesMediaAF::captionsStyleSheetOverride() const
550 {
551     if (testingMode())
552         return CaptionUserPreferences::captionsStyleSheetOverride();
553     
554     StringBuilder captionsOverrideStyleSheet;
555
556 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
557     if (!MediaAccessibilityLibrary())
558         return CaptionUserPreferences::captionsStyleSheetOverride();
559     
560     String captionsColor = captionsTextColorCSS();
561     String edgeStyle = captionsTextEdgeCSS();
562     String fontName = captionsDefaultFontCSS();
563     String background = captionsBackgroundCSS();
564     if (!background.isEmpty() || !captionsColor.isEmpty() || !edgeStyle.isEmpty() || !fontName.isEmpty()) {
565         captionsOverrideStyleSheet.appendLiteral(" video::");
566         captionsOverrideStyleSheet.append(TextTrackCue::cueShadowPseudoId());
567         captionsOverrideStyleSheet.append('{');
568         
569         if (!background.isEmpty())
570             captionsOverrideStyleSheet.append(background);
571         if (!captionsColor.isEmpty())
572             captionsOverrideStyleSheet.append(captionsColor);
573         if (!edgeStyle.isEmpty())
574             captionsOverrideStyleSheet.append(edgeStyle);
575         if (!fontName.isEmpty())
576             captionsOverrideStyleSheet.append(fontName);
577         
578         captionsOverrideStyleSheet.append('}');
579     }
580     
581     String windowColor = captionsWindowCSS();
582     String windowCornerRadius = windowRoundedCornerRadiusCSS();
583     if (!windowColor.isEmpty() || !windowCornerRadius.isEmpty()) {
584         captionsOverrideStyleSheet.appendLiteral(" video::");
585         captionsOverrideStyleSheet.append(VTTCue::cueBackdropShadowPseudoId());
586         captionsOverrideStyleSheet.append('{');
587         
588         if (!windowColor.isEmpty())
589             captionsOverrideStyleSheet.append(windowColor);
590         if (!windowCornerRadius.isEmpty()) {
591             captionsOverrideStyleSheet.append(windowCornerRadius);
592         }
593         
594         captionsOverrideStyleSheet.append('}');
595     }
596 #endif // HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
597
598     LOG(Media, "CaptionUserPreferencesMediaAF::captionsStyleSheetOverrideSetting style to:\n%s", captionsOverrideStyleSheet.toString().utf8().data());
599
600     return captionsOverrideStyleSheet.toString();
601 }
602
603 static String languageIdentifier(const String& languageCode)
604 {
605     if (languageCode.isEmpty())
606         return languageCode;
607
608     String lowercaseLanguageCode = languageCode.convertToASCIILowercase();
609
610     // Need 2U here to disambiguate String::operator[] from operator(NSString*, int)[] in a production build.
611     if (lowercaseLanguageCode.length() >= 3 && (lowercaseLanguageCode[2U] == '_' || lowercaseLanguageCode[2U] == '-'))
612         lowercaseLanguageCode.truncate(2);
613
614     return lowercaseLanguageCode;
615 }
616
617 static void buildDisplayStringForTrackBase(StringBuilder& displayName, const TrackBase& track)
618 {
619     String label = track.label();
620     String trackLanguageIdentifier = track.validBCP47Language();
621
622     RetainPtr<CFLocaleRef> currentLocale = adoptCF(CFLocaleCreate(kCFAllocatorDefault, defaultLanguage().createCFString().get()));
623     RetainPtr<CFStringRef> localeIdentifier = adoptCF(CFLocaleCreateCanonicalLocaleIdentifierFromString(kCFAllocatorDefault, trackLanguageIdentifier.createCFString().get()));
624     RetainPtr<CFStringRef> languageCF = adoptCF(CFLocaleCopyDisplayNameForPropertyValue(currentLocale.get(), kCFLocaleLanguageCode, localeIdentifier.get()));
625     String language = languageCF.get();
626
627     if (!label.isEmpty()) {
628         if (language.isEmpty() || label.contains(language))
629             displayName.append(label);
630         else {
631             RetainPtr<CFDictionaryRef> localeDict = adoptCF(CFLocaleCreateComponentsFromLocaleIdentifier(kCFAllocatorDefault, localeIdentifier.get()));
632             if (localeDict) {
633                 CFStringRef countryCode = 0;
634                 String countryName;
635                 
636                 CFDictionaryGetValueIfPresent(localeDict.get(), kCFLocaleCountryCode, (const void **)&countryCode);
637                 if (countryCode) {
638                     RetainPtr<CFStringRef> countryNameCF = adoptCF(CFLocaleCopyDisplayNameForPropertyValue(currentLocale.get(), kCFLocaleCountryCode, countryCode));
639                     countryName = countryNameCF.get();
640                 }
641                 
642                 if (!countryName.isEmpty())
643                     displayName.append(textTrackCountryAndLanguageMenuItemText(label, countryName, language));
644                 else
645                     displayName.append(textTrackLanguageMenuItemText(label, language));
646             }
647         }
648     } else {
649         String languageAndLocale = adoptCF(CFLocaleCopyDisplayNameForPropertyValue(currentLocale.get(), kCFLocaleIdentifier, trackLanguageIdentifier.createCFString().get())).get();
650         if (!languageAndLocale.isEmpty())
651             displayName.append(languageAndLocale);
652         else if (!language.isEmpty())
653             displayName.append(language);
654         else
655             displayName.append(localeIdentifier.get());
656     }
657 }
658
659 static String trackDisplayName(AudioTrack* track)
660 {
661     StringBuilder displayName;
662     buildDisplayStringForTrackBase(displayName, *track);
663     
664     if (displayName.isEmpty())
665         displayName.append(audioTrackNoLabelText());
666
667     if (track->kind() != AudioTrack::descriptionKeyword())
668         return displayName.toString();
669
670     return audioDescriptionTrackSuffixText(displayName.toString());
671 }
672
673 String CaptionUserPreferencesMediaAF::displayNameForTrack(AudioTrack* track) const
674 {
675     return trackDisplayName(track);
676 }
677
678 static String trackDisplayName(TextTrack* track)
679 {
680     if (track == TextTrack::captionMenuOffItem())
681         return textTrackOffMenuItemText();
682     if (track == TextTrack::captionMenuAutomaticItem())
683         return textTrackAutomaticMenuItemText();
684
685     StringBuilder displayNameBuilder;
686     buildDisplayStringForTrackBase(displayNameBuilder, *track);
687
688     if (displayNameBuilder.isEmpty())
689         displayNameBuilder.append(textTrackNoLabelText());
690
691     String displayName = displayNameBuilder.toString();
692
693     if (track->isClosedCaptions()) {
694         displayName = closedCaptionTrackMenuItemText(displayName);
695         if (track->isEasyToRead())
696             displayName = easyReaderTrackMenuItemText(displayName);
697
698         return displayName;
699     }
700
701     if (track->isSDH())
702         displayName = sdhTrackMenuItemText(displayName);
703
704     if (track->containsOnlyForcedSubtitles())
705         displayName = forcedTrackMenuItemText(displayName);
706
707     if (track->isEasyToRead())
708         displayName = easyReaderTrackMenuItemText(displayName);
709
710     return displayName;
711 }
712
713 String CaptionUserPreferencesMediaAF::displayNameForTrack(TextTrack* track) const
714 {
715     return trackDisplayName(track);
716 }
717
718 int CaptionUserPreferencesMediaAF::textTrackSelectionScore(TextTrack* track, HTMLMediaElement* mediaElement) const
719 {
720     CaptionDisplayMode displayMode = captionDisplayMode();
721     if (displayMode == Manual)
722         return 0;
723
724     bool legacyOverride = mediaElement->webkitClosedCaptionsVisible();
725     if (displayMode == AlwaysOn && (!userPrefersSubtitles() && !userPrefersCaptions() && !legacyOverride))
726         return 0;
727     if (track->kind() != TextTrack::Kind::Captions && track->kind() != TextTrack::Kind::Subtitles && track->kind() != TextTrack::Kind::Forced)
728         return 0;
729     if (!track->isMainProgramContent())
730         return 0;
731
732     bool trackHasOnlyForcedSubtitles = track->containsOnlyForcedSubtitles();
733     if (!legacyOverride && ((trackHasOnlyForcedSubtitles && displayMode != ForcedOnly) || (!trackHasOnlyForcedSubtitles && displayMode == ForcedOnly)))
734         return 0;
735
736     Vector<String> userPreferredCaptionLanguages = preferredLanguages();
737
738     if ((displayMode == Automatic && !legacyOverride) || trackHasOnlyForcedSubtitles) {
739
740         if (!mediaElement || !mediaElement->player())
741             return 0;
742
743         String textTrackLanguage = track->validBCP47Language();
744         if (textTrackLanguage.isEmpty())
745             return 0;
746
747         Vector<String> languageList;
748         languageList.reserveCapacity(1);
749
750         String audioTrackLanguage;
751         if (testingMode())
752             audioTrackLanguage = primaryAudioTrackLanguageOverride();
753         else
754             audioTrackLanguage = mediaElement->player()->languageOfPrimaryAudioTrack();
755
756         if (audioTrackLanguage.isEmpty())
757             return 0;
758
759         bool exactMatch;
760         if (trackHasOnlyForcedSubtitles) {
761             languageList.append(audioTrackLanguage);
762             size_t offset = indexOfBestMatchingLanguageInList(textTrackLanguage, languageList, exactMatch);
763
764             // Only consider a forced-only track if it IS in the same language as the primary audio track.
765             if (offset)
766                 return 0;
767         } else {
768             languageList.append(defaultLanguage());
769
770             // Only enable a text track if the current audio track is NOT in the user's preferred language ...
771             size_t offset = indexOfBestMatchingLanguageInList(audioTrackLanguage, languageList, exactMatch);
772             if (!offset)
773                 return 0;
774
775             // and the text track matches the user's preferred language.
776             offset = indexOfBestMatchingLanguageInList(textTrackLanguage, languageList, exactMatch);
777             if (offset)
778                 return 0;
779         }
780
781         userPreferredCaptionLanguages = languageList;
782     }
783
784     int trackScore = 0;
785
786     if (userPrefersCaptions()) {
787         // When the user prefers accessibility tracks, rank is SDH, then CC, then subtitles.
788         if (track->kind() == TextTrack::Kind::Subtitles)
789             trackScore = 1;
790         else if (track->isClosedCaptions())
791             trackScore = 2;
792         else
793             trackScore = 3;
794     } else {
795         // When the user prefers translation tracks, rank is subtitles, then SDH, then CC tracks.
796         if (track->kind() == TextTrack::Kind::Subtitles)
797             trackScore = 3;
798         else if (!track->isClosedCaptions())
799             trackScore = 2;
800         else
801             trackScore = 1;
802     }
803
804     return trackScore + textTrackLanguageSelectionScore(track, userPreferredCaptionLanguages);
805 }
806
807 static bool textTrackCompare(const RefPtr<TextTrack>& a, const RefPtr<TextTrack>& b)
808 {
809     String preferredLanguageDisplayName = displayNameForLanguageLocale(languageIdentifier(defaultLanguage()));
810     String aLanguageDisplayName = displayNameForLanguageLocale(languageIdentifier(a->validBCP47Language()));
811     String bLanguageDisplayName = displayNameForLanguageLocale(languageIdentifier(b->language()));
812
813     // Tracks in the user's preferred language are always at the top of the menu.
814     bool aIsPreferredLanguage = !codePointCompare(aLanguageDisplayName, preferredLanguageDisplayName);
815     bool bIsPreferredLanguage = !codePointCompare(bLanguageDisplayName, preferredLanguageDisplayName);
816     if ((aIsPreferredLanguage || bIsPreferredLanguage) && (aIsPreferredLanguage != bIsPreferredLanguage))
817         return aIsPreferredLanguage;
818
819     // Tracks not in the user's preferred language sort first by language ...
820     if (codePointCompare(aLanguageDisplayName, bLanguageDisplayName))
821         return codePointCompare(aLanguageDisplayName, bLanguageDisplayName) < 0;
822
823     // ... but when tracks have the same language, main program content sorts next highest ...
824     bool aIsMainContent = a->isMainProgramContent();
825     bool bIsMainContent = b->isMainProgramContent();
826     if ((aIsMainContent || bIsMainContent) && (aIsMainContent != bIsMainContent))
827         return aIsMainContent;
828
829     // ... and main program trakcs sort higher than CC tracks ...
830     bool aIsCC = a->isClosedCaptions();
831     bool bIsCC = b->isClosedCaptions();
832     if ((aIsCC || bIsCC) && (aIsCC != bIsCC)) {
833         if (aIsCC)
834             return aIsMainContent;
835         return bIsMainContent;
836     }
837
838     // ... and tracks of the same type and language sort by the menu item text.
839     return codePointCompare(trackDisplayName(a.get()), trackDisplayName(b.get())) < 0;
840 }
841
842 Vector<RefPtr<AudioTrack>> CaptionUserPreferencesMediaAF::sortedTrackListForMenu(AudioTrackList* trackList)
843 {
844     ASSERT(trackList);
845     
846     Vector<RefPtr<AudioTrack>> tracksForMenu;
847     
848     for (unsigned i = 0, length = trackList->length(); i < length; ++i) {
849         AudioTrack* track = trackList->item(i);
850         String language = displayNameForLanguageLocale(track->validBCP47Language());
851         tracksForMenu.append(track);
852     }
853     
854     std::sort(tracksForMenu.begin(), tracksForMenu.end(), [](auto& a, auto& b) {
855         return codePointCompare(trackDisplayName(a.get()), trackDisplayName(b.get())) < 0;
856     });
857     
858     return tracksForMenu;
859 }
860
861 Vector<RefPtr<TextTrack>> CaptionUserPreferencesMediaAF::sortedTrackListForMenu(TextTrackList* trackList)
862 {
863     ASSERT(trackList);
864
865     Vector<RefPtr<TextTrack>> tracksForMenu;
866     HashSet<String> languagesIncluded;
867     CaptionDisplayMode displayMode = captionDisplayMode();
868     bool prefersAccessibilityTracks = userPrefersCaptions();
869     bool filterTrackList = shouldFilterTrackMenu();
870
871     for (unsigned i = 0, length = trackList->length(); i < length; ++i) {
872         TextTrack* track = trackList->item(i);
873         String language = displayNameForLanguageLocale(track->validBCP47Language());
874
875         if (displayMode == Manual) {
876             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because selection mode is 'manual'", track->kindKeyword().string().utf8().data(), language.utf8().data());
877             tracksForMenu.append(track);
878             continue;
879         }
880
881         auto kind = track->kind();
882         if (kind != TextTrack::Kind::Captions && kind != TextTrack::Kind::Descriptions && kind != TextTrack::Kind::Subtitles)
883             continue;
884
885         if (track->containsOnlyForcedSubtitles()) {
886             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it contains only forced subtitles", track->kindKeyword().string().utf8().data(), language.utf8().data());
887             continue;
888         }
889         
890         if (track->isEasyToRead()) {
891             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because it is 'easy to read'", track->kindKeyword().string().utf8().data(), language.utf8().data());
892             if (!language.isEmpty())
893                 languagesIncluded.add(language);
894             tracksForMenu.append(track);
895             continue;
896         }
897
898         if (track->mode() == TextTrack::Mode::Showing) {
899             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because it is already visible", track->kindKeyword().string().utf8().data(), language.utf8().data());
900             if (!language.isEmpty())
901                 languagesIncluded.add(language);
902             tracksForMenu.append(track);
903             continue;
904         }
905
906         if (!language.isEmpty() && track->isMainProgramContent()) {
907             bool isAccessibilityTrack = track->kind() == TextTrack::Kind::Captions;
908             if (prefersAccessibilityTracks) {
909                 // In the first pass, include only caption tracks if the user prefers accessibility tracks.
910                 if (!isAccessibilityTrack && filterTrackList) {
911                     LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it is NOT an accessibility track", track->kindKeyword().string().utf8().data(), language.utf8().data());
912                     continue;
913                 }
914             } else {
915                 // In the first pass, only include the first non-CC or SDH track with each language if the user prefers translation tracks.
916                 if (isAccessibilityTrack && filterTrackList) {
917                     LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it is an accessibility track", track->kindKeyword().string().utf8().data(), language.utf8().data());
918                     continue;
919                 }
920                 if (languagesIncluded.contains(language) && filterTrackList) {
921                     LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it is not the first with this language", track->kindKeyword().string().utf8().data(), language.utf8().data());
922                     continue;
923                 }
924             }
925         }
926
927         if (!language.isEmpty())
928             languagesIncluded.add(language);
929         tracksForMenu.append(track);
930
931         LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s', is%s main program content", track->kindKeyword().string().utf8().data(), language.utf8().data(), track->isMainProgramContent() ? "" : " NOT");
932     }
933
934     // Now that we have filtered for the user's accessibility/translation preference, add  all tracks with a unique language without regard to track type.
935     for (unsigned i = 0, length = trackList->length(); i < length; ++i) {
936         TextTrack* track = trackList->item(i);
937         String language = displayNameForLanguageLocale(track->language());
938
939         if (tracksForMenu.contains(track))
940             continue;
941
942         auto kind = track->kind();
943         if (kind != TextTrack::Kind::Captions && kind != TextTrack::Kind::Descriptions && kind != TextTrack::Kind::Subtitles)
944             continue;
945
946         // All candidates with no languge were added the first time through.
947         if (language.isEmpty())
948             continue;
949
950         if (track->containsOnlyForcedSubtitles())
951             continue;
952
953         if (!languagesIncluded.contains(language) && track->isMainProgramContent()) {
954             languagesIncluded.add(language);
955             tracksForMenu.append(track);
956             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because it is the only track with this language", track->kindKeyword().string().utf8().data(), language.utf8().data());
957         }
958     }
959
960     if (tracksForMenu.isEmpty())
961         return tracksForMenu;
962
963     std::sort(tracksForMenu.begin(), tracksForMenu.end(), textTrackCompare);
964
965     tracksForMenu.insert(0, TextTrack::captionMenuOffItem());
966     tracksForMenu.insert(1, TextTrack::captionMenuAutomaticItem());
967
968     return tracksForMenu;
969 }
970     
971 }
972
973 #endif
974
975 #endif // ENABLE(VIDEO_TRACK)