50cf0e5f100fb586d6fc7d4df601aaeff2800096
[WebKit-https.git] / Source / WebCore / page / CaptionUserPreferencesMediaAF.cpp
1 /*
2  * Copyright (C) 2012-2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27
28 #if ENABLE(VIDEO_TRACK)
29
30 #if !USE(DIRECT2D)
31
32 #include "CaptionUserPreferencesMediaAF.h"
33
34 #include "AudioTrackList.h"
35 #include "FloatConversion.h"
36 #include "HTMLMediaElement.h"
37 #include "URL.h"
38 #include "Language.h"
39 #include "LocalizedStrings.h"
40 #include "Logging.h"
41 #include "MediaControlElements.h"
42 #include "SoftLinking.h"
43 #include "TextTrackList.h"
44 #include "UserStyleSheetTypes.h"
45 #include "VTTCue.h"
46 #include <wtf/NeverDestroyed.h>
47 #include <wtf/PlatformUserPreferredLanguages.h>
48 #include <wtf/RetainPtr.h>
49 #include <wtf/text/CString.h>
50 #include <wtf/text/StringBuilder.h>
51
52 #if PLATFORM(IOS)
53 #import "WebCoreThreadRun.h"
54 #endif
55
56 #if COMPILER(MSVC)
57 // See https://msdn.microsoft.com/en-us/library/35bhkfb6.aspx
58 #pragma warning(disable: 4273)
59 #endif
60
61 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
62 #include <CoreText/CoreText.h>
63 #include <MediaAccessibility/MediaAccessibility.h>
64
65 #include "MediaAccessibilitySoftLink.h"
66
67 #if PLATFORM(WIN)
68
69 #ifdef DEBUG_ALL
70 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_DEBUG_LIBRARY(Lib)
71 #else
72 #define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_LIBRARY(Lib)
73 #endif
74
75 #define SOFT_LINK_AVF(Lib, Name, Type) SOFT_LINK_DLL_IMPORT(Lib, Name, Type)
76 #define SOFT_LINK_AVF_POINTER(Lib, Name, Type) SOFT_LINK_VARIABLE_DLL_IMPORT_OPTIONAL(Lib, Name, Type)
77 #define SOFT_LINK_AVF_FRAMEWORK_IMPORT(Lib, Fun, ReturnType, Arguments, Signature) SOFT_LINK_DLL_IMPORT(Lib, Fun, ReturnType, __cdecl, Arguments, Signature)
78 #define SOFT_LINK_AVF_FRAMEWORK_IMPORT_OPTIONAL(Lib, Fun, ReturnType, Arguments) SOFT_LINK_DLL_IMPORT_OPTIONAL(Lib, Fun, ReturnType, __cdecl, Arguments)
79
80 // CoreText only needs to be soft-linked on Windows.
81 SOFT_LINK_AVF_FRAMEWORK(CoreText)
82 SOFT_LINK_AVF_FRAMEWORK_IMPORT(CoreText, CTFontDescriptorCopyAttribute,  CFTypeRef, (CTFontDescriptorRef descriptor, CFStringRef attribute), (descriptor, attribute));
83 SOFT_LINK_AVF_POINTER(CoreText, kCTFontNameAttribute, CFStringRef)
84 #define kCTFontNameAttribute getkCTFontNameAttribute()
85
86 #define CTFontDescriptorCopyAttribute softLink_CTFontDescriptorCopyAttribute
87
88 SOFT_LINK_AVF_FRAMEWORK(CoreMedia)
89 SOFT_LINK_AVF_FRAMEWORK_IMPORT_OPTIONAL(CoreMedia, MTEnableCaption2015Behavior, Boolean, ())
90
91 #else
92
93 SOFT_LINK_FRAMEWORK(MediaToolbox)
94 SOFT_LINK_OPTIONAL(MediaToolbox, MTEnableCaption2015Behavior, Boolean, (), ())
95
96 #endif // PLATFORM(WIN)
97
98 #endif // HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
99
100 namespace WebCore {
101
102 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
103 static void userCaptionPreferencesChangedNotificationCallback(CFNotificationCenterRef, void* observer, CFStringRef, const void *, CFDictionaryRef)
104 {
105 #if !PLATFORM(IOS)
106     static_cast<CaptionUserPreferencesMediaAF*>(observer)->captionPreferencesChanged();
107 #else
108     WebThreadRun(^{
109         static_cast<CaptionUserPreferencesMediaAF*>(observer)->captionPreferencesChanged();
110     });
111 #endif
112 }
113 #endif
114
115 CaptionUserPreferencesMediaAF::CaptionUserPreferencesMediaAF(PageGroup& group)
116     : CaptionUserPreferences(group)
117 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
118     , m_updateStyleSheetTimer(*this, &CaptionUserPreferencesMediaAF::updateTimerFired)
119     , m_listeningForPreferenceChanges(false)
120 #endif
121 {
122     static bool initialized;
123     if (!initialized) {
124         initialized = true;
125
126         MTEnableCaption2015BehaviorPtrType function = MTEnableCaption2015BehaviorPtr();
127         if (!function || !function())
128             return;
129
130         beginBlockingNotifications();
131         CaptionUserPreferences::setCaptionDisplayMode(Manual);
132         setUserPrefersCaptions(false);
133         setUserPrefersSubtitles(false);
134         setUserPrefersTextDescriptions(false);
135         endBlockingNotifications();
136     }
137 }
138
139 CaptionUserPreferencesMediaAF::~CaptionUserPreferencesMediaAF()
140 {
141 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
142     if (kMAXCaptionAppearanceSettingsChangedNotification)
143         CFNotificationCenterRemoveObserver(CFNotificationCenterGetLocalCenter(), this, kMAXCaptionAppearanceSettingsChangedNotification, 0);
144     if (kMAAudibleMediaSettingsChangedNotification)
145         CFNotificationCenterRemoveObserver(CFNotificationCenterGetLocalCenter(), this, kMAAudibleMediaSettingsChangedNotification, 0);
146 #endif
147 }
148
149 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
150
151 CaptionUserPreferences::CaptionDisplayMode CaptionUserPreferencesMediaAF::captionDisplayMode() const
152 {
153     CaptionDisplayMode internalMode = CaptionUserPreferences::captionDisplayMode();
154     if (internalMode == Manual || testingMode() || !MediaAccessibilityLibrary())
155         return internalMode;
156
157     MACaptionAppearanceDisplayType displayType = MACaptionAppearanceGetDisplayType(kMACaptionAppearanceDomainUser);
158     switch (displayType) {
159     case kMACaptionAppearanceDisplayTypeForcedOnly:
160         return ForcedOnly;
161
162     case kMACaptionAppearanceDisplayTypeAutomatic:
163         return Automatic;
164
165     case kMACaptionAppearanceDisplayTypeAlwaysOn:
166         return AlwaysOn;
167     }
168
169     ASSERT_NOT_REACHED();
170     return ForcedOnly;
171 }
172     
173 void CaptionUserPreferencesMediaAF::setCaptionDisplayMode(CaptionUserPreferences::CaptionDisplayMode mode)
174 {
175     if (testingMode() || !MediaAccessibilityLibrary()) {
176         CaptionUserPreferences::setCaptionDisplayMode(mode);
177         return;
178     }
179
180     if (captionDisplayMode() == Manual)
181         return;
182
183     MACaptionAppearanceDisplayType displayType = kMACaptionAppearanceDisplayTypeForcedOnly;
184     switch (mode) {
185     case Automatic:
186         displayType = kMACaptionAppearanceDisplayTypeAutomatic;
187         break;
188     case ForcedOnly:
189         displayType = kMACaptionAppearanceDisplayTypeForcedOnly;
190         break;
191     case AlwaysOn:
192         displayType = kMACaptionAppearanceDisplayTypeAlwaysOn;
193         break;
194     default:
195         ASSERT_NOT_REACHED();
196         break;
197     }
198
199     MACaptionAppearanceSetDisplayType(kMACaptionAppearanceDomainUser, displayType);
200 }
201
202 bool CaptionUserPreferencesMediaAF::userPrefersCaptions() const
203 {
204     bool captionSetting = CaptionUserPreferences::userPrefersCaptions();
205     if (captionSetting || testingMode() || !MediaAccessibilityLibrary())
206         return captionSetting;
207     
208     RetainPtr<CFArrayRef> captioningMediaCharacteristics = adoptCF(MACaptionAppearanceCopyPreferredCaptioningMediaCharacteristics(kMACaptionAppearanceDomainUser));
209     return captioningMediaCharacteristics && CFArrayGetCount(captioningMediaCharacteristics.get());
210 }
211
212 bool CaptionUserPreferencesMediaAF::userPrefersSubtitles() const
213 {
214     bool subtitlesSetting = CaptionUserPreferences::userPrefersSubtitles();
215     if (subtitlesSetting || testingMode() || !MediaAccessibilityLibrary())
216         return subtitlesSetting;
217     
218     RetainPtr<CFArrayRef> captioningMediaCharacteristics = adoptCF(MACaptionAppearanceCopyPreferredCaptioningMediaCharacteristics(kMACaptionAppearanceDomainUser));
219     return !(captioningMediaCharacteristics && CFArrayGetCount(captioningMediaCharacteristics.get()));
220 }
221
222 void CaptionUserPreferencesMediaAF::updateTimerFired()
223 {
224     updateCaptionStyleSheetOverride();
225 }
226
227 void CaptionUserPreferencesMediaAF::setInterestedInCaptionPreferenceChanges()
228 {
229     if (m_listeningForPreferenceChanges)
230         return;
231
232     if (!MediaAccessibilityLibrary())
233         return;
234
235     if (!kMAXCaptionAppearanceSettingsChangedNotification && !canLoad_MediaAccessibility_kMAAudibleMediaSettingsChangedNotification())
236         return;
237
238     m_listeningForPreferenceChanges = true;
239     m_registeringForNotification = true;
240
241     if (kMAXCaptionAppearanceSettingsChangedNotification)
242         CFNotificationCenterAddObserver(CFNotificationCenterGetLocalCenter(), this, userCaptionPreferencesChangedNotificationCallback, kMAXCaptionAppearanceSettingsChangedNotification, 0, CFNotificationSuspensionBehaviorCoalesce);
243     if (canLoad_MediaAccessibility_kMAAudibleMediaSettingsChangedNotification())
244         CFNotificationCenterAddObserver(CFNotificationCenterGetLocalCenter(), this, userCaptionPreferencesChangedNotificationCallback, kMAAudibleMediaSettingsChangedNotification, 0, CFNotificationSuspensionBehaviorCoalesce);
245     m_registeringForNotification = false;
246
247     // Generating and registering the caption stylesheet can be expensive and this method is called indirectly when the parser creates an audio or
248     // video element, so do it after a brief pause.
249     m_updateStyleSheetTimer.startOneShot(0);
250 }
251
252 void CaptionUserPreferencesMediaAF::captionPreferencesChanged()
253 {
254     if (m_registeringForNotification)
255         return;
256
257     if (m_listeningForPreferenceChanges)
258         updateCaptionStyleSheetOverride();
259
260     CaptionUserPreferences::captionPreferencesChanged();
261 }
262
263 String CaptionUserPreferencesMediaAF::captionsWindowCSS() const
264 {
265     MACaptionAppearanceBehavior behavior;
266     RetainPtr<CGColorRef> color = adoptCF(MACaptionAppearanceCopyWindowColor(kMACaptionAppearanceDomainUser, &behavior));
267
268     Color windowColor(color.get());
269     if (!windowColor.isValid())
270         windowColor = Color::transparent;
271
272     bool important = behavior == kMACaptionAppearanceBehaviorUseValue;
273     CGFloat opacity = MACaptionAppearanceGetWindowOpacity(kMACaptionAppearanceDomainUser, &behavior);
274     if (!important)
275         important = behavior == kMACaptionAppearanceBehaviorUseValue;
276     String windowStyle = colorPropertyCSS(CSSPropertyBackgroundColor, Color(windowColor.red(), windowColor.green(), windowColor.blue(), static_cast<int>(opacity * 255)), important);
277
278     if (!opacity)
279         return windowStyle;
280
281     return makeString(windowStyle, getPropertyNameString(CSSPropertyPadding), ": .4em !important;");
282 }
283
284 String CaptionUserPreferencesMediaAF::captionsBackgroundCSS() const
285 {
286     // This default value must be the same as the one specified in mediaControls.css for -webkit-media-text-track-past-nodes
287     // and webkit-media-text-track-future-nodes.
288     static NeverDestroyed<Color> defaultBackgroundColor(0, 0, 0, 0.8 * 255);
289
290     MACaptionAppearanceBehavior behavior;
291
292     RetainPtr<CGColorRef> color = adoptCF(MACaptionAppearanceCopyBackgroundColor(kMACaptionAppearanceDomainUser, &behavior));
293     Color backgroundColor(color.get());
294     if (!backgroundColor.isValid())
295         backgroundColor = defaultBackgroundColor;
296
297     bool important = behavior == kMACaptionAppearanceBehaviorUseValue;
298     CGFloat opacity = MACaptionAppearanceGetBackgroundOpacity(kMACaptionAppearanceDomainUser, &behavior);
299     if (!important)
300         important = behavior == kMACaptionAppearanceBehaviorUseValue;
301     return colorPropertyCSS(CSSPropertyBackgroundColor, Color(backgroundColor.red(), backgroundColor.green(), backgroundColor.blue(), static_cast<int>(opacity * 255)), important);
302 }
303
304 Color CaptionUserPreferencesMediaAF::captionsTextColor(bool& important) const
305 {
306     MACaptionAppearanceBehavior behavior;
307     RetainPtr<CGColorRef> color = adoptCF(MACaptionAppearanceCopyForegroundColor(kMACaptionAppearanceDomainUser, &behavior));
308     Color textColor(color.get());
309     if (!textColor.isValid())
310         // This default value must be the same as the one specified in mediaControls.css for -webkit-media-text-track-container.
311         textColor = Color::white;
312     
313     important = behavior == kMACaptionAppearanceBehaviorUseValue;
314     CGFloat opacity = MACaptionAppearanceGetForegroundOpacity(kMACaptionAppearanceDomainUser, &behavior);
315     if (!important)
316         important = behavior == kMACaptionAppearanceBehaviorUseValue;
317     return Color(textColor.red(), textColor.green(), textColor.blue(), static_cast<int>(opacity * 255));
318 }
319     
320 String CaptionUserPreferencesMediaAF::captionsTextColorCSS() const
321 {
322     bool important;
323     Color textColor = captionsTextColor(important);
324
325     if (!textColor.isValid())
326         return emptyString();
327
328     return colorPropertyCSS(CSSPropertyColor, textColor, important);
329 }
330     
331 String CaptionUserPreferencesMediaAF::windowRoundedCornerRadiusCSS() const
332 {
333     MACaptionAppearanceBehavior behavior;
334     CGFloat radius = MACaptionAppearanceGetWindowRoundedCornerRadius(kMACaptionAppearanceDomainUser, &behavior);
335     if (!radius)
336         return emptyString();
337
338     StringBuilder builder;
339     builder.append(getPropertyNameString(CSSPropertyBorderRadius));
340     builder.append(String::format(":%.02fpx", radius));
341     if (behavior == kMACaptionAppearanceBehaviorUseValue)
342         builder.appendLiteral(" !important");
343     builder.append(';');
344
345     return builder.toString();
346 }
347     
348 Color CaptionUserPreferencesMediaAF::captionsEdgeColorForTextColor(const Color& textColor) const
349 {
350     int distanceFromWhite = differenceSquared(textColor, Color::white);
351     int distanceFromBlack = differenceSquared(textColor, Color::black);
352     
353     if (distanceFromWhite < distanceFromBlack)
354         return textColor.dark();
355     
356     return textColor.light();
357 }
358
359 String CaptionUserPreferencesMediaAF::cssPropertyWithTextEdgeColor(CSSPropertyID id, const String& value, const Color& textColor, bool important) const
360 {
361     StringBuilder builder;
362     
363     builder.append(getPropertyNameString(id));
364     builder.append(':');
365     builder.append(value);
366     builder.append(' ');
367     builder.append(captionsEdgeColorForTextColor(textColor).serialized());
368     if (important)
369         builder.appendLiteral(" !important");
370     builder.append(';');
371     
372     return builder.toString();
373 }
374
375 String CaptionUserPreferencesMediaAF::colorPropertyCSS(CSSPropertyID id, const Color& color, bool important) const
376 {
377     StringBuilder builder;
378     
379     builder.append(getPropertyNameString(id));
380     builder.append(':');
381     builder.append(color.serialized());
382     if (important)
383         builder.appendLiteral(" !important");
384     builder.append(';');
385     
386     return builder.toString();
387 }
388
389 String CaptionUserPreferencesMediaAF::strokeWidth() const
390 {
391     static NeverDestroyed<const String> strokeWidthDefault(ASCIILiteral(" .03em "));
392     
393     if (!MACaptionFontAttributeStrokeWidth && !canLoad_MediaAccessibility_MACaptionFontAttributeStrokeWidth())
394         return strokeWidthDefault;
395     
396     MACaptionAppearanceBehavior behavior;
397
398     auto font = adoptCF(MACaptionAppearanceCopyFontDescriptorForStyle(kMACaptionAppearanceDomainUser, &behavior, kMACaptionAppearanceFontStyleDefault));
399     if (!font)
400         return strokeWidthDefault;
401     
402     auto strokeWidthAttribute = adoptCF(CTFontDescriptorCopyAttribute(font.get(), MACaptionFontAttributeStrokeWidth));
403     if (!strokeWidthAttribute)
404         return strokeWidthDefault;
405     
406     int strokeWidth = 0;
407     if (!CFNumberGetValue(static_cast<CFNumberRef>(strokeWidthAttribute.get()), kCFNumberIntType, &strokeWidth))
408         return strokeWidthDefault;
409
410     return String::format(" %dpx ", strokeWidth);
411 }
412
413 String CaptionUserPreferencesMediaAF::captionsTextEdgeCSS() const
414 {
415     static NeverDestroyed<const String> edgeStyleRaised(ASCIILiteral(" -.05em -.05em 0 "));
416     static NeverDestroyed<const String> edgeStyleDepressed(ASCIILiteral(" .05em .05em 0 "));
417     static NeverDestroyed<const String> edgeStyleDropShadow(ASCIILiteral(" .075em .075em 0 "));
418
419     bool unused;
420     Color color = captionsTextColor(unused);
421     if (!color.isValid())
422         color = Color { Color::black };
423     color = captionsEdgeColorForTextColor(color);
424
425     MACaptionAppearanceBehavior behavior;
426     MACaptionAppearanceTextEdgeStyle textEdgeStyle = MACaptionAppearanceGetTextEdgeStyle(kMACaptionAppearanceDomainUser, &behavior);
427     switch (textEdgeStyle) {
428     case kMACaptionAppearanceTextEdgeStyleUndefined:
429     case kMACaptionAppearanceTextEdgeStyleNone:
430         return emptyString();
431             
432     case kMACaptionAppearanceTextEdgeStyleRaised:
433         return cssPropertyWithTextEdgeColor(CSSPropertyTextShadow, edgeStyleRaised, color, behavior == kMACaptionAppearanceBehaviorUseValue);
434     case kMACaptionAppearanceTextEdgeStyleDepressed:
435         return cssPropertyWithTextEdgeColor(CSSPropertyTextShadow, edgeStyleDepressed, color, behavior == kMACaptionAppearanceBehaviorUseValue);
436     case kMACaptionAppearanceTextEdgeStyleDropShadow:
437         return cssPropertyWithTextEdgeColor(CSSPropertyTextShadow, edgeStyleDropShadow, color, behavior == kMACaptionAppearanceBehaviorUseValue);
438     case kMACaptionAppearanceTextEdgeStyleUniform:
439         return cssPropertyWithTextEdgeColor(CSSPropertyWebkitTextStroke, strokeWidth(), color, behavior == kMACaptionAppearanceBehaviorUseValue);
440     
441     default:
442         ASSERT_NOT_REACHED();
443         break;
444     }
445     
446     return emptyString();
447 }
448
449 String CaptionUserPreferencesMediaAF::captionsDefaultFontCSS() const
450 {
451     MACaptionAppearanceBehavior behavior;
452     
453     RetainPtr<CTFontDescriptorRef> font = adoptCF(MACaptionAppearanceCopyFontDescriptorForStyle(kMACaptionAppearanceDomainUser, &behavior, kMACaptionAppearanceFontStyleDefault));
454     if (!font)
455         return emptyString();
456
457     RetainPtr<CFTypeRef> name = adoptCF(CTFontDescriptorCopyAttribute(font.get(), kCTFontNameAttribute));
458     if (!name)
459         return emptyString();
460     
461     StringBuilder builder;
462     
463     builder.append(getPropertyNameString(CSSPropertyFontFamily));
464     builder.appendLiteral(": \"");
465     builder.append(static_cast<CFStringRef>(name.get()));
466     builder.append('"');
467     if (behavior == kMACaptionAppearanceBehaviorUseValue)
468         builder.appendLiteral(" !important");
469     builder.append(';');
470     
471     return builder.toString();
472 }
473
474 float CaptionUserPreferencesMediaAF::captionFontSizeScaleAndImportance(bool& important) const
475 {
476     if (testingMode() || !MediaAccessibilityLibrary())
477         return CaptionUserPreferences::captionFontSizeScaleAndImportance(important);
478
479     MACaptionAppearanceBehavior behavior;
480     CGFloat characterScale = CaptionUserPreferences::captionFontSizeScaleAndImportance(important);
481     CGFloat scaleAdjustment = MACaptionAppearanceGetRelativeCharacterSize(kMACaptionAppearanceDomainUser, &behavior);
482
483     if (!scaleAdjustment)
484         return characterScale;
485
486     important = behavior == kMACaptionAppearanceBehaviorUseValue;
487 #if defined(__LP64__) && __LP64__
488     return narrowPrecisionToFloat(scaleAdjustment * characterScale);
489 #else
490     return scaleAdjustment * characterScale;
491 #endif
492 }
493
494 void CaptionUserPreferencesMediaAF::setPreferredLanguage(const String& language)
495 {
496     if (CaptionUserPreferences::captionDisplayMode() == Manual)
497         return;
498
499     if (testingMode() || !MediaAccessibilityLibrary()) {
500         CaptionUserPreferences::setPreferredLanguage(language);
501         return;
502     }
503
504     MACaptionAppearanceAddSelectedLanguage(kMACaptionAppearanceDomainUser, language.createCFString().get());
505 }
506
507 Vector<String> CaptionUserPreferencesMediaAF::preferredLanguages() const
508 {
509     if (testingMode() || !MediaAccessibilityLibrary())
510         return CaptionUserPreferences::preferredLanguages();
511
512     Vector<String> platformLanguages = platformUserPreferredLanguages();
513     Vector<String> override = userPreferredLanguagesOverride();
514     if (!override.isEmpty()) {
515         if (platformLanguages.size() != override.size())
516             return override;
517         for (size_t i = 0; i < override.size(); i++) {
518             if (override[i] != platformLanguages[i])
519                 return override;
520         }
521     }
522
523     CFIndex languageCount = 0;
524     RetainPtr<CFArrayRef> languages = adoptCF(MACaptionAppearanceCopySelectedLanguages(kMACaptionAppearanceDomainUser));
525     if (languages)
526         languageCount = CFArrayGetCount(languages.get());
527
528     if (!languageCount)
529         return CaptionUserPreferences::preferredLanguages();
530
531     Vector<String> userPreferredLanguages;
532     userPreferredLanguages.reserveCapacity(languageCount + platformLanguages.size());
533     for (CFIndex i = 0; i < languageCount; i++)
534         userPreferredLanguages.append(static_cast<CFStringRef>(CFArrayGetValueAtIndex(languages.get(), i)));
535
536     userPreferredLanguages.appendVector(platformLanguages);
537
538     return userPreferredLanguages;
539 }
540
541 void CaptionUserPreferencesMediaAF::setPreferredAudioCharacteristic(const String& characteristic)
542 {
543     if (testingMode() || !MediaAccessibilityLibrary())
544         CaptionUserPreferences::setPreferredAudioCharacteristic(characteristic);
545 }
546
547 Vector<String> CaptionUserPreferencesMediaAF::preferredAudioCharacteristics() const
548 {
549     if (testingMode() || !MediaAccessibilityLibrary() || !canLoad_MediaAccessibility_MAAudibleMediaCopyPreferredCharacteristics())
550         return CaptionUserPreferences::preferredAudioCharacteristics();
551
552     CFIndex characteristicCount = 0;
553     RetainPtr<CFArrayRef> characteristics = adoptCF(MAAudibleMediaCopyPreferredCharacteristics());
554     if (characteristics)
555         characteristicCount = CFArrayGetCount(characteristics.get());
556
557     if (!characteristicCount)
558         return CaptionUserPreferences::preferredAudioCharacteristics();
559
560     Vector<String> userPreferredAudioCharacteristics;
561     userPreferredAudioCharacteristics.reserveCapacity(characteristicCount);
562     for (CFIndex i = 0; i < characteristicCount; i++)
563         userPreferredAudioCharacteristics.append(static_cast<CFStringRef>(CFArrayGetValueAtIndex(characteristics.get(), i)));
564
565     return userPreferredAudioCharacteristics;
566 }
567 #endif // HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
568
569 String CaptionUserPreferencesMediaAF::captionsStyleSheetOverride() const
570 {
571     if (testingMode())
572         return CaptionUserPreferences::captionsStyleSheetOverride();
573     
574     StringBuilder captionsOverrideStyleSheet;
575
576 #if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
577     if (!MediaAccessibilityLibrary())
578         return CaptionUserPreferences::captionsStyleSheetOverride();
579     
580     String captionsColor = captionsTextColorCSS();
581     String edgeStyle = captionsTextEdgeCSS();
582     String fontName = captionsDefaultFontCSS();
583     String background = captionsBackgroundCSS();
584     if (!background.isEmpty() || !captionsColor.isEmpty() || !edgeStyle.isEmpty() || !fontName.isEmpty()) {
585         captionsOverrideStyleSheet.appendLiteral(" video::");
586         captionsOverrideStyleSheet.append(TextTrackCue::cueShadowPseudoId());
587         captionsOverrideStyleSheet.append('{');
588         
589         if (!background.isEmpty())
590             captionsOverrideStyleSheet.append(background);
591         if (!captionsColor.isEmpty())
592             captionsOverrideStyleSheet.append(captionsColor);
593         if (!edgeStyle.isEmpty())
594             captionsOverrideStyleSheet.append(edgeStyle);
595         if (!fontName.isEmpty())
596             captionsOverrideStyleSheet.append(fontName);
597         
598         captionsOverrideStyleSheet.append('}');
599     }
600     
601     String windowColor = captionsWindowCSS();
602     String windowCornerRadius = windowRoundedCornerRadiusCSS();
603     if (!windowColor.isEmpty() || !windowCornerRadius.isEmpty()) {
604         captionsOverrideStyleSheet.appendLiteral(" video::");
605         captionsOverrideStyleSheet.append(VTTCue::cueBackdropShadowPseudoId());
606         captionsOverrideStyleSheet.append('{');
607         
608         if (!windowColor.isEmpty())
609             captionsOverrideStyleSheet.append(windowColor);
610         if (!windowCornerRadius.isEmpty()) {
611             captionsOverrideStyleSheet.append(windowCornerRadius);
612         }
613         
614         captionsOverrideStyleSheet.append('}');
615     }
616 #endif // HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
617
618     LOG(Media, "CaptionUserPreferencesMediaAF::captionsStyleSheetOverrideSetting sytle to:\n%s", captionsOverrideStyleSheet.toString().utf8().data());
619
620     return captionsOverrideStyleSheet.toString();
621 }
622
623 static String languageIdentifier(const String& languageCode)
624 {
625     if (languageCode.isEmpty())
626         return languageCode;
627
628     String lowercaseLanguageCode = languageCode.convertToASCIILowercase();
629
630     // Need 2U here to disambiguate String::operator[] from operator(NSString*, int)[] in a production build.
631     if (lowercaseLanguageCode.length() >= 3 && (lowercaseLanguageCode[2U] == '_' || lowercaseLanguageCode[2U] == '-'))
632         lowercaseLanguageCode.truncate(2);
633
634     return lowercaseLanguageCode;
635 }
636
637 static void buildDisplayStringForTrackBase(StringBuilder& displayName, const TrackBase& track)
638 {
639     String label = track.label();
640     String trackLanguageIdentifier = track.validBCP47Language();
641
642     RetainPtr<CFLocaleRef> currentLocale = adoptCF(CFLocaleCreate(kCFAllocatorDefault, defaultLanguage().createCFString().get()));
643     RetainPtr<CFStringRef> localeIdentifier = adoptCF(CFLocaleCreateCanonicalLocaleIdentifierFromString(kCFAllocatorDefault, trackLanguageIdentifier.createCFString().get()));
644     RetainPtr<CFStringRef> languageCF = adoptCF(CFLocaleCopyDisplayNameForPropertyValue(currentLocale.get(), kCFLocaleLanguageCode, localeIdentifier.get()));
645     String language = languageCF.get();
646
647     if (!label.isEmpty()) {
648         if (language.isEmpty() || label.contains(language))
649             displayName.append(label);
650         else {
651             RetainPtr<CFDictionaryRef> localeDict = adoptCF(CFLocaleCreateComponentsFromLocaleIdentifier(kCFAllocatorDefault, localeIdentifier.get()));
652             if (localeDict) {
653                 CFStringRef countryCode = 0;
654                 String countryName;
655                 
656                 CFDictionaryGetValueIfPresent(localeDict.get(), kCFLocaleCountryCode, (const void **)&countryCode);
657                 if (countryCode) {
658                     RetainPtr<CFStringRef> countryNameCF = adoptCF(CFLocaleCopyDisplayNameForPropertyValue(currentLocale.get(), kCFLocaleCountryCode, countryCode));
659                     countryName = countryNameCF.get();
660                 }
661                 
662                 if (!countryName.isEmpty())
663                     displayName.append(textTrackCountryAndLanguageMenuItemText(label, countryName, language));
664                 else
665                     displayName.append(textTrackLanguageMenuItemText(label, language));
666             }
667         }
668     } else {
669         String languageAndLocale = adoptCF(CFLocaleCopyDisplayNameForPropertyValue(currentLocale.get(), kCFLocaleIdentifier, trackLanguageIdentifier.createCFString().get())).get();
670         if (!languageAndLocale.isEmpty())
671             displayName.append(languageAndLocale);
672         else if (!language.isEmpty())
673             displayName.append(language);
674         else
675             displayName.append(localeIdentifier.get());
676     }
677 }
678
679 static String trackDisplayName(AudioTrack* track)
680 {
681     StringBuilder displayName;
682     buildDisplayStringForTrackBase(displayName, *track);
683     
684     if (displayName.isEmpty())
685         displayName.append(audioTrackNoLabelText());
686
687     if (track->kind() != AudioTrack::descriptionKeyword())
688         return displayName.toString();
689
690     return audioDescriptionTrackSuffixText(displayName.toString());
691 }
692
693 String CaptionUserPreferencesMediaAF::displayNameForTrack(AudioTrack* track) const
694 {
695     return trackDisplayName(track);
696 }
697
698 static String trackDisplayName(TextTrack* track)
699 {
700     if (track == TextTrack::captionMenuOffItem())
701         return textTrackOffMenuItemText();
702     if (track == TextTrack::captionMenuAutomaticItem())
703         return textTrackAutomaticMenuItemText();
704
705     StringBuilder displayNameBuilder;
706     buildDisplayStringForTrackBase(displayNameBuilder, *track);
707
708     if (displayNameBuilder.isEmpty())
709         displayNameBuilder.append(textTrackNoLabelText());
710
711     String displayName = displayNameBuilder.toString();
712
713     if (track->isClosedCaptions()) {
714         displayName = closedCaptionTrackMenuItemText(displayName);
715         if (track->isEasyToRead())
716             displayName = easyReaderTrackMenuItemText(displayName);
717
718         return displayName;
719     }
720
721     if (track->isSDH())
722         displayName = sdhTrackMenuItemText(displayName);
723
724     if (track->containsOnlyForcedSubtitles())
725         displayName = forcedTrackMenuItemText(displayName);
726
727     if (track->isEasyToRead())
728         displayName = easyReaderTrackMenuItemText(displayName);
729
730     return displayName;
731 }
732
733 String CaptionUserPreferencesMediaAF::displayNameForTrack(TextTrack* track) const
734 {
735     return trackDisplayName(track);
736 }
737
738 int CaptionUserPreferencesMediaAF::textTrackSelectionScore(TextTrack* track, HTMLMediaElement* mediaElement) const
739 {
740     CaptionDisplayMode displayMode = captionDisplayMode();
741     if (displayMode == Manual)
742         return 0;
743
744     bool legacyOverride = mediaElement->webkitClosedCaptionsVisible();
745     if (displayMode == AlwaysOn && (!userPrefersSubtitles() && !userPrefersCaptions() && !legacyOverride))
746         return 0;
747     if (track->kind() != TextTrack::Kind::Captions && track->kind() != TextTrack::Kind::Subtitles && track->kind() != TextTrack::Kind::Forced)
748         return 0;
749     if (!track->isMainProgramContent())
750         return 0;
751
752     bool trackHasOnlyForcedSubtitles = track->containsOnlyForcedSubtitles();
753     if (!legacyOverride && ((trackHasOnlyForcedSubtitles && displayMode != ForcedOnly) || (!trackHasOnlyForcedSubtitles && displayMode == ForcedOnly)))
754         return 0;
755
756     Vector<String> userPreferredCaptionLanguages = preferredLanguages();
757
758     if ((displayMode == Automatic && !legacyOverride) || trackHasOnlyForcedSubtitles) {
759
760         if (!mediaElement || !mediaElement->player())
761             return 0;
762
763         String textTrackLanguage = track->validBCP47Language();
764         if (textTrackLanguage.isEmpty())
765             return 0;
766
767         Vector<String> languageList;
768         languageList.reserveCapacity(1);
769
770         String audioTrackLanguage;
771         if (testingMode())
772             audioTrackLanguage = primaryAudioTrackLanguageOverride();
773         else
774             audioTrackLanguage = mediaElement->player()->languageOfPrimaryAudioTrack();
775
776         if (audioTrackLanguage.isEmpty())
777             return 0;
778
779         bool exactMatch;
780         if (trackHasOnlyForcedSubtitles) {
781             languageList.append(audioTrackLanguage);
782             size_t offset = indexOfBestMatchingLanguageInList(textTrackLanguage, languageList, exactMatch);
783
784             // Only consider a forced-only track if it IS in the same language as the primary audio track.
785             if (offset)
786                 return 0;
787         } else {
788             languageList.append(defaultLanguage());
789
790             // Only enable a text track if the current audio track is NOT in the user's preferred language ...
791             size_t offset = indexOfBestMatchingLanguageInList(audioTrackLanguage, languageList, exactMatch);
792             if (!offset)
793                 return 0;
794
795             // and the text track matches the user's preferred language.
796             offset = indexOfBestMatchingLanguageInList(textTrackLanguage, languageList, exactMatch);
797             if (offset)
798                 return 0;
799         }
800
801         userPreferredCaptionLanguages = languageList;
802     }
803
804     int trackScore = 0;
805
806     if (userPrefersCaptions()) {
807         // When the user prefers accessibility tracks, rank is SDH, then CC, then subtitles.
808         if (track->kind() == TextTrack::Kind::Subtitles)
809             trackScore = 1;
810         else if (track->isClosedCaptions())
811             trackScore = 2;
812         else
813             trackScore = 3;
814     } else {
815         // When the user prefers translation tracks, rank is subtitles, then SDH, then CC tracks.
816         if (track->kind() == TextTrack::Kind::Subtitles)
817             trackScore = 3;
818         else if (!track->isClosedCaptions())
819             trackScore = 2;
820         else
821             trackScore = 1;
822     }
823
824     return trackScore + textTrackLanguageSelectionScore(track, userPreferredCaptionLanguages);
825 }
826
827 static bool textTrackCompare(const RefPtr<TextTrack>& a, const RefPtr<TextTrack>& b)
828 {
829     String preferredLanguageDisplayName = displayNameForLanguageLocale(languageIdentifier(defaultLanguage()));
830     String aLanguageDisplayName = displayNameForLanguageLocale(languageIdentifier(a->validBCP47Language()));
831     String bLanguageDisplayName = displayNameForLanguageLocale(languageIdentifier(b->language()));
832
833     // Tracks in the user's preferred language are always at the top of the menu.
834     bool aIsPreferredLanguage = !codePointCompare(aLanguageDisplayName, preferredLanguageDisplayName);
835     bool bIsPreferredLanguage = !codePointCompare(bLanguageDisplayName, preferredLanguageDisplayName);
836     if ((aIsPreferredLanguage || bIsPreferredLanguage) && (aIsPreferredLanguage != bIsPreferredLanguage))
837         return aIsPreferredLanguage;
838
839     // Tracks not in the user's preferred language sort first by language ...
840     if (codePointCompare(aLanguageDisplayName, bLanguageDisplayName))
841         return codePointCompare(aLanguageDisplayName, bLanguageDisplayName) < 0;
842
843     // ... but when tracks have the same language, main program content sorts next highest ...
844     bool aIsMainContent = a->isMainProgramContent();
845     bool bIsMainContent = b->isMainProgramContent();
846     if ((aIsMainContent || bIsMainContent) && (aIsMainContent != bIsMainContent))
847         return aIsMainContent;
848
849     // ... and main program trakcs sort higher than CC tracks ...
850     bool aIsCC = a->isClosedCaptions();
851     bool bIsCC = b->isClosedCaptions();
852     if ((aIsCC || bIsCC) && (aIsCC != bIsCC)) {
853         if (aIsCC)
854             return aIsMainContent;
855         return bIsMainContent;
856     }
857
858     // ... and tracks of the same type and language sort by the menu item text.
859     return codePointCompare(trackDisplayName(a.get()), trackDisplayName(b.get())) < 0;
860 }
861
862 Vector<RefPtr<AudioTrack>> CaptionUserPreferencesMediaAF::sortedTrackListForMenu(AudioTrackList* trackList)
863 {
864     ASSERT(trackList);
865     
866     Vector<RefPtr<AudioTrack>> tracksForMenu;
867     
868     for (unsigned i = 0, length = trackList->length(); i < length; ++i) {
869         AudioTrack* track = trackList->item(i);
870         String language = displayNameForLanguageLocale(track->validBCP47Language());
871         tracksForMenu.append(track);
872     }
873     
874     std::sort(tracksForMenu.begin(), tracksForMenu.end(), [](auto& a, auto& b) {
875         return codePointCompare(trackDisplayName(a.get()), trackDisplayName(b.get())) < 0;
876     });
877     
878     return tracksForMenu;
879 }
880
881 Vector<RefPtr<TextTrack>> CaptionUserPreferencesMediaAF::sortedTrackListForMenu(TextTrackList* trackList)
882 {
883     ASSERT(trackList);
884
885     Vector<RefPtr<TextTrack>> tracksForMenu;
886     HashSet<String> languagesIncluded;
887     CaptionDisplayMode displayMode = captionDisplayMode();
888     bool prefersAccessibilityTracks = userPrefersCaptions();
889     bool filterTrackList = shouldFilterTrackMenu();
890
891     for (unsigned i = 0, length = trackList->length(); i < length; ++i) {
892         TextTrack* track = trackList->item(i);
893         String language = displayNameForLanguageLocale(track->validBCP47Language());
894
895         if (displayMode == Manual) {
896             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because selection mode is 'manual'", track->kindKeyword().string().utf8().data(), language.utf8().data());
897             tracksForMenu.append(track);
898             continue;
899         }
900
901         auto kind = track->kind();
902         if (kind != TextTrack::Kind::Captions && kind != TextTrack::Kind::Descriptions && kind != TextTrack::Kind::Subtitles)
903             continue;
904
905         if (track->containsOnlyForcedSubtitles()) {
906             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it contains only forced subtitles", track->kindKeyword().string().utf8().data(), language.utf8().data());
907             continue;
908         }
909         
910         if (track->isEasyToRead()) {
911             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because it is 'easy to read'", track->kindKeyword().string().utf8().data(), language.utf8().data());
912             if (!language.isEmpty())
913                 languagesIncluded.add(language);
914             tracksForMenu.append(track);
915             continue;
916         }
917
918         if (track->mode() == TextTrack::Mode::Showing) {
919             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because it is already visible", track->kindKeyword().string().utf8().data(), language.utf8().data());
920             if (!language.isEmpty())
921                 languagesIncluded.add(language);
922             tracksForMenu.append(track);
923             continue;
924         }
925
926         if (!language.isEmpty() && track->isMainProgramContent()) {
927             bool isAccessibilityTrack = track->kind() == TextTrack::Kind::Captions;
928             if (prefersAccessibilityTracks) {
929                 // In the first pass, include only caption tracks if the user prefers accessibility tracks.
930                 if (!isAccessibilityTrack && filterTrackList) {
931                     LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it is NOT an accessibility track", track->kindKeyword().string().utf8().data(), language.utf8().data());
932                     continue;
933                 }
934             } else {
935                 // In the first pass, only include the first non-CC or SDH track with each language if the user prefers translation tracks.
936                 if (isAccessibilityTrack && filterTrackList) {
937                     LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it is an accessibility track", track->kindKeyword().string().utf8().data(), language.utf8().data());
938                     continue;
939                 }
940                 if (languagesIncluded.contains(language) && filterTrackList) {
941                     LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - skipping '%s' track with language '%s' because it is not the first with this language", track->kindKeyword().string().utf8().data(), language.utf8().data());
942                     continue;
943                 }
944             }
945         }
946
947         if (!language.isEmpty())
948             languagesIncluded.add(language);
949         tracksForMenu.append(track);
950
951         LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s', is%s main program content", track->kindKeyword().string().utf8().data(), language.utf8().data(), track->isMainProgramContent() ? "" : " NOT");
952     }
953
954     // Now that we have filtered for the user's accessibility/translation preference, add  all tracks with a unique language without regard to track type.
955     for (unsigned i = 0, length = trackList->length(); i < length; ++i) {
956         TextTrack* track = trackList->item(i);
957         String language = displayNameForLanguageLocale(track->language());
958
959         if (tracksForMenu.contains(track))
960             continue;
961
962         auto kind = track->kind();
963         if (kind != TextTrack::Kind::Captions && kind != TextTrack::Kind::Descriptions && kind != TextTrack::Kind::Subtitles)
964             continue;
965
966         // All candidates with no languge were added the first time through.
967         if (language.isEmpty())
968             continue;
969
970         if (track->containsOnlyForcedSubtitles())
971             continue;
972
973         if (!languagesIncluded.contains(language) && track->isMainProgramContent()) {
974             languagesIncluded.add(language);
975             tracksForMenu.append(track);
976             LOG(Media, "CaptionUserPreferencesMediaAF::sortedTrackListForMenu - adding '%s' track with language '%s' because it is the only track with this language", track->kindKeyword().string().utf8().data(), language.utf8().data());
977         }
978     }
979
980     if (tracksForMenu.isEmpty())
981         return tracksForMenu;
982
983     std::sort(tracksForMenu.begin(), tracksForMenu.end(), textTrackCompare);
984
985     tracksForMenu.insert(0, TextTrack::captionMenuOffItem());
986     tracksForMenu.insert(1, TextTrack::captionMenuAutomaticItem());
987
988     return tracksForMenu;
989 }
990     
991 }
992
993 #endif
994
995 #endif // ENABLE(VIDEO_TRACK)