A recap of video and audio
Desktop apps use
IMFMediaEngine,
etc.
Multimedia platform
Windows.Media
namespace
Great battery life
with hardware
offload
Apps can extend
formats and codecs
Audio/video
reader (source)
Video
decoder
Video
effect 1
Video
effect n
Video
encoder
Audio
decoder
Audio
effect 1
Audio
effect n
Audio
encoder
Audio/video
writer or render
(sink)
Content protection, e.g. PlayReady
Background audio
Stream to Xbox, TVs and audio receivers
Convert between popular formats
Low latency mode for communications
and games
Video stabilization
Zoom and mirror modes
Stereo 3D
And more
Improved quality, performance & battery
Adaptive bitrate and encrypted streaming
with HTML5 standards
Improved online playlists and ad insertion
in connected standby
Custom formats without native code
Unconstrained download of large files
App discovery and launching on Xbox One
Audio effect management
Edit media file metadata in local libraries
And more
Simple background audio playlist
// Create audio element that continues to play in the background
<audio id=“audioplayer" msAudioCategory=“BackgroundCapableMedia” autoplay />
// First track is from local library, then 2nd is chosen by the app
var picker = new Windows.Storage.FileOpenPicker();
picker.pickSingleFileAsync().
then(function(file) {
var url = URL.createObjectURL(file);
document.getElementById(“audioplayer”).src = url;
});
audio.addEventListener("ended", function() {
document.getElementById(“audioplayer”).src = "http://contoso.com/track2.m4a";
}); /* If the device idles to connected standby, track2 can’t be fetched */
Audio playlist while in connected standby
// Create downloader and download instance for a given URI to download to “newFile”
var nextItem = "http://contoso.com/track2.m4a";
var downloader = new Windows.Networking.BackgroundTransfer.BackgroundDownloader();
var download = downloader.createDownload(nextItem, newFile);
// Set priority so the download starts immediately
download.priority =
Windows.Networking.BackgroundTransfer.BackgroundTransferPriority.high;
download.startAsync();
// Play the resulting file
var url = URL.createObjectURL(newFile);
document.getElementById(“audioplayer”).src = url;
Media playback controls
function initSystemMediaControls() { /* initialize at app activation */
SystemMediaControls =
Windows.Media.SystemMediaTransportControls.getForCurrentView();
SystemMediaControls. /* Events for both software and hardware buttons */
addEventListener(“buttonpressed”, systemMediaControlsButtonPressed, false);
SystemMediaControls.isPlayEnabled = true;
/* Enable all buttons */
SystemMediaControls.isPauseEnabled = true;
// Reflect the current playback state in the system UI
SystemMediaControls.playbackStatus = Windows.Media.MediaPlaybackStatus.closed;
}
Media playback controls
function systemMediaControlsButtonPressed(e) {
var mediaButton = Windows.Media.SystemMediaTransportControlsButton;
switch (e.button) {
case mediaButton.play: audio.play(); break;
/* Add case statements for
/* each button */
case mediaButton.pause: audio.pause(); break;
}
}
function updateSystemMediaTransportControlsMetadata() {
displayUpdater.type = Windows.Media.MediaPlaybackType.music;
musicProperties.title = “my track title”; /* Add other metadata too */
displayUpdater.update();
}
Stream to, and remotely control Xbox
// Step 1: Obtain PlayToManager object for app’s current view
var ptm = Windows.Media.PlayToManager.getForCurrentView();
// Step 2: Register for the “sourcerequested” event (when user selects Devices charm)
ptm.addEventListener(“sourcerequested”, function(e) {
var request = e.sourceRequest;
var deferral = request.getDeferral();
// Step 3: Specify the media to be streamed
request.setSource(document.getElementById(“audioplayer”).msPlayToSource);
deferral.complete();
// The media will then be streamed to the device chosen by the user in the Charm.
});
Clarity on what devices are supported:
Play, Print, Project
Add a device from the charm
Adobe Flash video and audio now works
from the browser
Audio from a video can be streamed to
audio-only receivers (e.g. music videos)
Add a custom file or stream format
App is responsible
for retrieving audio
samples and
parsing file format
MediaStreamSource
takes elementary
stream samples as
input
Decoding and
rendering by
platform
Multimedia platform
Get audio samples (e.g. sockets)
and parse file format
Windows.Media.Core.
MediaStreamSource
Audio/video
reader (source)
Media Stream
Source
Video
decoder
Video
effect 1
Video
effect n
Video
encoder
Audio
decoder
Audio
effect 1
Audio
effect n
Audio
encoder
Audio/video
writer (sink)
Add a custom file or stream format
// Audio sample details and encoding properties:
// Raw AAC data, 48KHz sampling rate, 2 channel and approx 124Kbps
var sampleSize = 1024;
var sampleDuration = 20;
var audioProps
= Windows.Media.MediaProperties.AudioEncodingProperties.
createAac(48000, 2, 124338);
// Create the media source with specific encoding properties.
var streamSource = new Windows.Media.Core.MediaStreamSource(new
Windows.Media.Core.AudioStreamDescriptor(audioProps));
Add a custom file or stream format
// Register handlers
streamSource.addEventListener(“samplerequested”, sampleRequestedHandler, false);
streamSource.canSeek = true;
streamSource.musicProperties.title = “My track title”;
var songDuration = 60000;
// Attach media source to the audio tag and start playing
var audio = document.getElementById(“audiotag”);
audio.src = URL.createObjectURL(streamSource);
audio.play();
Add a custom file or stream format
function sampleRequestedHandler(e) {
var request = e.request;
if (!MyCustomFormatParser.IsAtEndOfStream(randomAccessStream)) {
var deferral = request.getDeferral();
var inputStream = MyCustomFormatParser.getInputStream(randomAccessStream);
MediaStreamSample.createFromStreamAsync(inputStream, sampleSize, timeOffset).
then(function(sample) {
});
}
}
sample.duration = sampleDuration;
/* When sample is requested, your
*/
sample.keyFrame = true;
/* custom parser extracts audio data
*/
request.sample = sample;
/* from RandomAccessStream and returns
*/
deferral.complete();
/* an InputStream of only audio samples */
Add audio declaration in app manifest and
set msAudioCategory attribute
Handle media button events for play and
pause at minimum
Set high priority on background
downloader for online playlists
PlayTo app contract for streaming to Xbox
Use MediaStreamSource for advanced
streaming or custom formats
Simple video playback, Windows 8.1
<MediaElement Name=“VideoPlayer”
Source=“http://contoso.com/movie1.m4v”
PosterSource=“movie1_artwork.png”
AreTransportControlsEnabled=“true” />
/* No longer necessary to create buttons and handlers for each.
All transport control buttons, including Zoom and Fullscreen
are free, as are power and performance optimizations */
Simple video playback, before 8.1
<MediaElement Name=“VideoPlayer”
Source=“http://contoso.com/movie1.m4v”
PosterSource=“movie1_artwork.png” />
/* Before 8.1, Media playback buttons had to be manual.
You can still do this for custom controls */
<StackPanel x:Name=“mtcPanel” Orientation=“Horizontal”>
<Button Name=“btnPlay” Click=“btnPlay_Click” Content=“Play” />
<Button Name=“btnStop” Click=“btnStop_Click” Content=“Stop” />
<Button Name=“btnZoom” Click=“btnZoom_Click” Content=“Zoom” />
<Button Name=“btnFull” Click=“btnFull_Click” Content=“Full” />
</StackPanel>
Simple video playback, before 8.1
private DisplayRequest dispRequest = null;
private void btnPlay_Click(object sender, RoutedEventArgs e)
{
VideoPlayer.Play();
/* Before 8.1, button handlers
*/
if (dispRequest == null)
/* had to be manual, and a
*/
{
/* display request had to be made */
dispRequest = new DisplayRequest();
/* to prevent the screen from
*/
dispRequest.RequestActive();
/* turning off after 2 minutes
*/
}
}
private void btnStop_Click(object sender, RoutedEventArgs e)
{
VideoPlayer.Stop();
if (dispRequest != null) { dispRequest.RequestRelease(); }
}
Full screen video, Windows 8.1
/* Full screen is free using AreTransportControlsEnabled.
This hides all UI and stretches to screen perfectly to ensure hardware offload */
<MediaElement Name=“VideoPlayer”
Source=“http://contoso.com/movie1.m4v”
PosterSource=“movie1_artwork.png”
AreTransportControlsEnabled=“true” />
/* For custom controls, let Windows optimize for hardware offload
and set display request using the new IsFullWindow */
private void btnFull_Click(object sender, RoutedEventArgs e)
{
VideoPlayer.IsFullWindow = “true”;
}
Video zoom for small screen tablets
/* Zoom is free using AreTransportControlsEnabled */
<MediaElement Name=“VideoPlayer”
Source=“http://contoso.com/movie1.m4v”
PosterSource=“movie1_artwork.png”
AreTransportControlsEnabled=“true” />
/* For custom controls, use Stretch */
private void btnZoom_Click(object sender, RoutedEventArgs e)
{
VideoPlayer.Stretch = “UniformToFill”;
}
Media playback controls
private SystemMediaTransportControls SystemMediaControls = null;
public void InitSystemMediaControls()
{
SystemMediaControls = SystemMediaTransportControls.getForCurrentView();
/* The new SystemMediaTransportControls events for software and hardware buttons */
SystemMediaControls.ButtonPressed += SystemMediaControls_ButtonPressed;
/* Enable specific media playback buttons (for brevity, only play and pause are shown) */
SystemMediaControls.IsPlayEnabled = true;
SystemMediaControls.IsPauseEnabled = true;
/* Reflect the current playback state in the system UI */
SystemMediaControls.PlaybackStatus = MediaPlaybackStatus.Closed;
}
Media playback controls
private async void SystemMediaControls_ButtonPressed(SystemMediaTransportControls sender,
SystemMediaTransportControlsButtonPressedEventArgs args)
{
switch
{
case SystemMediaTransportControlsButton.Play:
VideoPlayer.Play(); break;
}
/* Add case statements for
*/
/* each button – both system UI
*/
/* and hardware buttons will event the same */
}
/* Sets metadata from properties in the passed StorageFile */
private async Task UpdateSystemMediaControlsDisplayAsync(StorageFile mediaFile)
{
copyFromFileAsyncSuccessful = await
SystemMediaControls.DisplayUpdater.CopyFromFileAsync(Music, mediaFile);
}
Mute or pause when app is not visible
<MediaElement Name=“VideoPlayer”
Source=“http://contoso.com/movie1.m4v”
PosterSource=“movie1_artwork.png”
AudioCategory=“ForegroundOnlyMedia” /> /* Audio will mute when app */
/* is not visible
/* If pause is appropriate when the video is not in view,
listen for sound level changes similar to listening for button presses */
SystemMediaTransportControls SystemMediaControls =
SystemMediaTransportControls.getForCurrentView();
SystemMediaControls.PropertyChanged += SystemMediaControls_PropertyChanged;
*/
Mute or pause when app is not visible
private async void SystemMediaControls_PropertyChanged(SystemMediaTransportControls sender,
SystemMediaTransportControlsPropertyChangedEventArgs args)
{
if (args.Property == SystemMediaTransportControlsProperty.SoundLevel)
{
if (sender.SoundLevel == SoundLevel.Muted)
{
/* SoundLevel will only report ‘Muted’ when the app is not visible */
VideoPlayer.Pause();
}
else
{
/* If SoundLevel isn’t ‘Muted’ the app must be visible */
VideoPlayer.Play();
}
}
}
Unconstrained download
StorageFile offlineMovie = /* path for saving downloaded file */;
BackgroundDownloader downloader = new BackgroundTransfer.BackgroundDownloader();
var download = downloader.CreateDownload(new
Windows.Foundation.Uri(“http://contoso.com/movie1.m4v”), offlineMovie);
List<DownloadOperation> operations = new List<DownloadOperation>();
operations.Add(download); /* Add as many downloads as you like */
var result = await BackgroundDownloader.RequestUnconstrainedDownloadsAsync(operations);
if (result.IsUnconstrained)
{ /* If user accepts prompt, downloads are unconstrained. If no; constrained */
}
Download.StartAsync().AsTask( /* Implement progress indicator, etc. */ );
Stream to, and remotely control Xbox
/* Step 1: Obtain PlayToManager object for app’s current view */
PlayToManager ptm = PlayToManager.GetForCurrentView();
/* Step 2: Register for SourceRequested event (fires when Devices charm selected) */
ptm.SourceRequested += (PlayToManager sender, PlayToSourceRequestedEventArgs e) =>
{
request = e.SourceRequest;
PlayToSourceDeferral deferral = request.GetDeferral();
request.SetSource(VideoPlayer.PlayToSource); /* Step 3: Set source of media */
deferral.Complete();
}
/* The media will then be streamed to the device chosen by the user */
Add a custom file or stream format
/* Audio sample details and encoding properties:
Raw AAC data, 48KHz sampling rate, 2 channel and approx 124Kbps */
AudioEncodingProperties AudioProps
=
new MediaProperties.AudioEncodingProperties.CreateAac(48000, 2, 124338);
/* Create the media source with specific encoding properties. */
MediaStreamSource StreamSource = new Windows.Media.Core.MediaStreamSource(new
Windows.Media.Core.AudioStreamDescriptor(AudioProps));
Add a custom file or stream format
/* Handle events on the media source. Only SampleRequested is shown for brevity */
StreamSource.SampleRequested += StreamSource_SampleRequested;
/* Set higher level properties */
StreamSource.canSeek = true;
StreamSource.MusicProperties.title = “My track title”;
TimeSpan songDuration = new TimeSpan(600000000);
StreamSource.duration = SongDuration;
/* Attach media source to XAML MediaElement and start playing */
AudioPlayer.SetMediaStreamSource(StreamSource);
AudioPlayer.Play();
Add a custom file or stream format
async void StreamSource_SampleRequested(MediaStreamSource sender, MediaStreamSourceSampleRequestedEventArgs args)
{
MediaStreamSourceSampleRequest request = args.Request;
/* Whenever a sample is requested,
*/
/* MyCustomFormatParser parses the
*/
if (!MyCustomFormatParser.IsAtEndOfStream(RandomAccessStream)) /* RandomAccessStream to retrieve only raw audio */
{
/* data then returns it as an InputStream
*/
Windows.Media.Core.StartingRequestDeferral deferral = request.GetDeferral();
IInputStream InputStream = MyCustomFormatParser.GetInputStream(RandomAccessStream);
MediaStreamSample Sample = await MediaStreamSample.CreateFromStreamAsync(InputStream, sampleSize, timeOffset);
Sample.duration = SampleDuration;
Sample.keyFrame = true;
timeOffset.Ticks += SampleDuration.Ticks;
Request.Sample = Sample;
deferral.Complete();
}
}
AreTransportControlsEnabled for
integrated controls and best performance
With custom controls, optimize for full
screen using IsFullWindow
Handle media button events for play and
pause at minimum
SoundLevelChanged events to pause when
not visible
Unconstrained background downloader
for large files
PlayTo app contract for streaming to Xbox
MediaStreamSource for advanced
streaming and custom formats
Video is offloaded to GPU, even when not
full screen
Best performance when full screen
Delay setting source URI until ready to play
If you control video encoding, match
encoded resolution with device resolutions
Follow touch target size guidelines
Support video zoom for letterbox removal
Consider playing full screen as default
Consider locking screen orientation to
landscape when playing full screen
Xbox One app manifest
<!-- Register ms-playtoapp protocol handler to enable app launching.
Apps can register multiple protocol handlers -->
<Extensions>
<Extension Category=“windows.protocol” StartPage=“main.html”>
<Protocol Name=“ms-playtoapp-myapp” />
</Extension>
</Extensions>
Windows 8.1 app
/* Pass custom meta info used to initiate streaming from Xbox One app
using x-ms-playToPreferredUri */
<video id=“player”
src=“http://contoso.com/movie.m4v”
x-ms-playToPreferredUri=“ms-playtoapp-myapp://media?id=12345”
controls />
Xbox One app contract
ptreceiver = new Windows.Media.PlayTo.PlayToReceiver();
/* Receiver accepts commands */
ptreceiver.supportsVideo = true;
/* from home network, and
ptreceiver.friendlyName = “My Movies App”;
/* advertises playback state */
*/
// Subscribe to events on the PlayToReceiver so you know when to control the video tag
ptreceiver.addEventListener("playrequested", playRequestedHandler, false);
ptreceiver.addEventListener("sourcechangerequested", sourceChangeRequestedHandler, false);
// Subscribe to events on the video tag so you can update playback state on home network
videoplayer.addEventListener("playing“, playingStateHandler, false);
// Advertise the receiver on the home network and start receiving playback commands
ptreceiver.startAsync().then( /* notify of current volume level, etc. */ );
Xbox One app contract
function playRequestedHandler(e) { document.getElementById(“videoplayer”).play(); }
function sourceChangeRequestedHandler(e) { /* Set the source based on whether a preferred URI
if (e.stream) {
/* is provided or not */
if (e.properties["System.ItemUrl"] != null &&
e.properties["System.ItemUrl"].indexOf("ms-playtoapp") > -1) {
// Parse custom Uri to extract variables encoded to initiate streaming
else {
var blob = MSApp.createBlobFromRandomAccessStream(e.stream.contentType, e.stream);
document.getElementById(“videoplayer").src = URL.createObjectURL(blob);
}
}
}
/* If x-ms-playToPreferredUri isn’t specified, just set source normally */
Xbox One app contract
/* Call associated notifying method to update the home network whenever the state
of the video tag change, such as: volumechange, ratechange, loadedmetadata,
durationchange, seeking, seeked, playing, pause, ended, error */
function playingStateHandler(e) {
ptreceiver.notifyPlaying();
}
Designing great entertainment apps for Windows
Playing and previewing audio and video
Creating a media player app
Handling hardware media buttons