feat(web): add support for casting (#18231)

* recreate #13966

* gcast button works

* rewrote gcast-player to be GCastDestination and CastManager manages the interface between UI and casting destinations

* remove unneeded imports

* add "Connected to" translation

* Remove css for cast launcher

* fix tests

* fix doc tests

* fix the receiver application ID

* remove casting app ID

* remove cast button from nav bar

It is now present at the following locations:

- shared link album and single asset views
- asset viewer (normal user)
- album view (normal user)

* part 1 of fixes from @danieldietzler code review

* part 2 of code review changes from @danieldietzler and @jsram91

* cleanup documentation

* onVideoStarted missing callback

* add token expiry validation

* cleanup logic and logging

* small cleanup

* rename to ICastDestination

* cast button changes
This commit is contained in:
Brandon Wees
2025-05-20 16:08:23 -05:00
committed by GitHub
parent 12b7a079c1
commit 86db0aafe5
16 changed files with 708 additions and 36 deletions

View File

@@ -0,0 +1,45 @@
<script lang="ts">
import { t } from 'svelte-i18n';
import { onMount } from 'svelte';
import { mdiCast, mdiCastConnected } from '@mdi/js';
import CircleIconButton from '$lib/components/elements/buttons/circle-icon-button.svelte';
import { CastDestinationType, castManager } from '$lib/managers/cast-manager.svelte';
import { GCastDestination } from '$lib/utils/cast/gcast-destination.svelte';
import { IconButton } from '@immich/ui';
interface Props {
whiteHover?: boolean;
navBar?: boolean;
}
let { whiteHover, navBar }: Props = $props();
onMount(async () => {
await castManager.initialize();
});
const getButtonColor = () => {
return castManager.isCasting ? 'primary' : whiteHover ? undefined : 'opaque';
};
</script>
{#if castManager.availableDestinations.length > 0 && castManager.availableDestinations[0].type === CastDestinationType.GCAST}
{#if navBar}
<IconButton
shape="round"
variant="ghost"
size="medium"
color={castManager.isCasting ? 'primary' : 'secondary'}
icon={castManager.isCasting ? mdiCastConnected : mdiCast}
onclick={() => void GCastDestination.showCastDialog()}
aria-label={$t('cast')}
/>
{:else}
<CircleIconButton
color={getButtonColor()}
icon={castManager.isCasting ? mdiCastConnected : mdiCast}
onclick={GCastDestination.showCastDialog}
title={$t('cast')}
/>
{/if}
{/if}

View File

@@ -21,6 +21,7 @@
import ImmichLogoSmallLink from '../shared-components/immich-logo-small-link.svelte';
import ThemeButton from '../shared-components/theme-button.svelte';
import AlbumSummary from './album-summary.svelte';
import CastButton from '$lib/cast/cast-button.svelte';
interface Props {
sharedLink: SharedLinkResponseDto;
@@ -103,6 +104,8 @@
{/snippet}
{#snippet trailing()}
<CastButton whiteHover />
{#if sharedLink.allowUpload}
<CircleIconButton
title={$t('add_photos')}

View File

@@ -1,5 +1,6 @@
<script lang="ts">
import { goto } from '$app/navigation';
import CastButton from '$lib/cast/cast-button.svelte';
import type { OnAction, PreAction } from '$lib/components/asset-viewer/actions/action';
import AddToAlbumAction from '$lib/components/asset-viewer/actions/add-to-album-action.svelte';
import ArchiveAction from '$lib/components/asset-viewer/actions/archive-action.svelte';
@@ -116,6 +117,8 @@
{/if}
</div>
<div class="flex gap-2 overflow-x-auto text-white" data-testid="asset-viewer-navbar-actions">
<CastButton />
{#if !asset.isTrashed && $user && !isLocked}
<ShareAction {asset} />
{/if}

View File

@@ -31,6 +31,29 @@ describe('PhotoViewer component', () => {
beforeAll(() => {
getAssetOriginalUrlSpy = vi.spyOn(utils, 'getAssetOriginalUrl');
getAssetThumbnailUrlSpy = vi.spyOn(utils, 'getAssetThumbnailUrl');
vi.stubGlobal('cast', {
framework: {
CastState: {
NO_DEVICES_AVAILABLE: 'NO_DEVICES_AVAILABLE',
},
RemotePlayer: vi.fn().mockImplementation(() => ({})),
RemotePlayerEventType: {
ANY_CHANGE: 'anyChanged',
},
RemotePlayerController: vi.fn().mockImplementation(() => ({ addEventListener: vi.fn() })),
CastContext: {
getInstance: vi.fn().mockImplementation(() => ({ setOptions: vi.fn(), addEventListener: vi.fn() })),
},
CastContextEventType: {
SESSION_STATE_CHANGED: 'sessionstatechanged',
CAST_STATE_CHANGED: 'caststatechanged',
},
},
});
vi.stubGlobal('chrome', {
cast: { media: { PlayerState: { IDLE: 'IDLE' } }, AutoJoinPolicy: { ORIGIN_SCOPED: 'origin_scoped' } },
});
});
beforeEach(() => {

View File

@@ -23,6 +23,7 @@
import { fade } from 'svelte/transition';
import LoadingSpinner from '../shared-components/loading-spinner.svelte';
import { NotificationType, notificationController } from '../shared-components/notification/notification';
import { castManager } from '$lib/managers/cast-manager.svelte';
interface Props {
asset: AssetResponseDto;
@@ -147,6 +148,27 @@
return AssetMediaSize.Preview;
});
$effect(() => {
if (assetFileUrl) {
// this can't be in an async context with $effect
void cast(assetFileUrl);
}
});
const cast = async (url: string) => {
if (!url || !castManager.isCasting) {
return;
}
const fullUrl = new URL(url, globalThis.location.href);
try {
await castManager.loadMedia(fullUrl.href);
} catch (error) {
handleError(error, 'Unable to cast');
return;
}
};
const onload = () => {
imageLoaded = true;
assetFileUrl = imageLoaderUrl;

View File

@@ -1,6 +1,8 @@
<script lang="ts">
import FaceEditor from '$lib/components/asset-viewer/face-editor/face-editor.svelte';
import VideoRemoteViewer from '$lib/components/asset-viewer/video-remote-viewer.svelte';
import LoadingSpinner from '$lib/components/shared-components/loading-spinner.svelte';
import { castManager } from '$lib/managers/cast-manager.svelte';
import { isFaceEditMode } from '$lib/stores/face-edit.svelte';
import { loopVideo as loopVideoPreference, videoViewerMuted, videoViewerVolume } from '$lib/stores/preferences.store';
import { getAssetPlaybackUrl, getAssetThumbnailUrl } from '$lib/utils';
@@ -41,8 +43,8 @@
let isScrubbing = $state(false);
onMount(() => {
assetFileUrl = getAssetPlaybackUrl({ id: assetId, cacheKey });
if (videoPlayer) {
assetFileUrl = getAssetPlaybackUrl({ id: assetId, cacheKey });
forceMuted = false;
videoPlayer.load();
}
@@ -106,42 +108,53 @@
bind:clientWidth={containerWidth}
bind:clientHeight={containerHeight}
>
<video
bind:this={videoPlayer}
loop={$loopVideoPreference && loopVideo}
autoplay
playsinline
controls
class="h-full object-contain"
use:swipe={() => ({})}
onswipe={onSwipe}
oncanplay={(e) => handleCanPlay(e.currentTarget)}
onended={onVideoEnded}
onvolumechange={(e) => {
if (!forceMuted) {
$videoViewerMuted = e.currentTarget.muted;
}
}}
onseeking={() => (isScrubbing = true)}
onseeked={() => (isScrubbing = false)}
onplaying={(e) => {
e.currentTarget.focus();
}}
onclose={() => onClose()}
muted={forceMuted || $videoViewerMuted}
bind:volume={$videoViewerVolume}
poster={getAssetThumbnailUrl({ id: assetId, size: AssetMediaSize.Preview, cacheKey })}
src={assetFileUrl}
>
</video>
{#if isLoading}
<div class="absolute flex place-content-center place-items-center">
<LoadingSpinner />
{#if castManager.isCasting}
<div class="place-content-center h-full place-items-center">
<VideoRemoteViewer
poster={getAssetThumbnailUrl({ id: assetId, size: AssetMediaSize.Preview, cacheKey })}
{onVideoStarted}
{onVideoEnded}
{assetFileUrl}
/>
</div>
{/if}
{:else}
<video
bind:this={videoPlayer}
loop={$loopVideoPreference && loopVideo}
autoplay
playsinline
controls
class="h-full object-contain"
use:swipe={() => ({})}
onswipe={onSwipe}
oncanplay={(e) => handleCanPlay(e.currentTarget)}
onended={onVideoEnded}
onvolumechange={(e) => {
if (!forceMuted) {
$videoViewerMuted = e.currentTarget.muted;
}
}}
onseeking={() => (isScrubbing = true)}
onseeked={() => (isScrubbing = false)}
onplaying={(e) => {
e.currentTarget.focus();
}}
onclose={() => onClose()}
muted={forceMuted || $videoViewerMuted}
bind:volume={$videoViewerVolume}
poster={getAssetThumbnailUrl({ id: assetId, size: AssetMediaSize.Preview, cacheKey })}
src={assetFileUrl}
>
</video>
{#if isFaceEditMode.value}
<FaceEditor htmlElement={videoPlayer} {containerWidth} {containerHeight} {assetId} />
{#if isLoading}
<div class="absolute flex place-content-center place-items-center">
<LoadingSpinner />
</div>
{/if}
{#if isFaceEditMode.value}
<FaceEditor htmlElement={videoPlayer} {containerWidth} {containerHeight} {assetId} />
{/if}
{/if}
</div>

View File

@@ -0,0 +1,103 @@
<script lang="ts">
import CircleIconButton from '$lib/components/elements/buttons/circle-icon-button.svelte';
import Icon from '$lib/components/elements/icon.svelte';
import LoadingSpinner from '$lib/components/shared-components/loading-spinner.svelte';
import { castManager, CastState } from '$lib/managers/cast-manager.svelte';
import { handleError } from '$lib/utils/handle-error';
import { mdiCastConnected, mdiPause, mdiPlay } from '@mdi/js';
import { t } from 'svelte-i18n';
interface Props {
poster: string;
assetFileUrl: string;
onVideoStarted: () => void;
onVideoEnded: () => void;
}
let { poster, assetFileUrl, onVideoEnded, onVideoStarted }: Props = $props();
let previousPlayerState: CastState | null = $state(null);
const handlePlayPauseButton = async () => {
switch (castManager.castState) {
case CastState.PLAYING: {
castManager.pause();
break;
}
case CastState.IDLE: {
await cast(assetFileUrl, true);
break;
}
default: {
castManager.play();
break;
}
}
};
$effect(() => {
if (assetFileUrl) {
// this can't be in an async context with $effect
void cast(assetFileUrl);
}
});
$effect(() => {
if (castManager.castState === CastState.IDLE && previousPlayerState !== CastState.PAUSED) {
onVideoEnded();
}
previousPlayerState = castManager.castState;
});
const cast = async (url: string, force: boolean = false) => {
if (!url || !castManager.isCasting) {
return;
}
const fullUrl = new URL(url, globalThis.location.href);
try {
await castManager.loadMedia(fullUrl.href, force);
onVideoStarted();
} catch (error) {
handleError(error, 'Unable to cast');
return;
}
};
function handleSeek(event: Event) {
const newTime = Number.parseFloat((event.target as HTMLInputElement).value);
castManager.seekTo(newTime);
}
</script>
<span class="flex items-center space-x-2 text-gray-200 text-2xl font-bold">
<Icon path={mdiCastConnected} class="text-primary" size="36" />
<span>{$t('connected_to')} {castManager.receiverName}</span>
</span>
<img src={poster} alt="poster" class="rounded-xl m-4" />
<div class="flex place-content-center place-items-center">
{#if castManager.castState == CastState.BUFFERING}
<div class="p-3">
<LoadingSpinner />
</div>
{:else}
<CircleIconButton
color="opaque"
icon={castManager.castState == CastState.PLAYING ? mdiPause : mdiPlay}
onclick={() => handlePlayPauseButton()}
title={castManager.castState == CastState.PLAYING ? 'Pause' : 'Play'}
/>
{/if}
<input
type="range"
min="0"
max={castManager.duration}
value={castManager.currentTime ?? 0}
onchange={handleSeek}
class="w-full h-4 bg-primary"
/>
</div>

View File

@@ -27,6 +27,7 @@
import ThemeButton from '../theme-button.svelte';
import UserAvatar from '../user-avatar.svelte';
import AccountInfoPanel from './account-info-panel.svelte';
import CastButton from '$lib/cast/cast-button.svelte';
interface Props {
showUploadButton?: boolean;
@@ -162,6 +163,8 @@
{/if}
</div>
<CastButton navBar />
<div
use:clickOutside={{
onOutclick: () => (shouldShowAccountInfoPanel = false),

View File

@@ -0,0 +1,159 @@
import { GCastDestination } from '$lib/utils/cast/gcast-destination.svelte';
import { createSession, type SessionCreateResponseDto } from '@immich/sdk';
import { DateTime, Duration } from 'luxon';
// follows chrome.cast.media.PlayerState
export enum CastState {
IDLE = 'IDLE',
PLAYING = 'PLAYING',
PAUSED = 'PAUSED',
BUFFERING = 'BUFFERING',
}
export enum CastDestinationType {
GCAST = 'GCAST',
}
export interface ICastDestination {
initialize(): Promise<boolean>; // returns if the cast destination can be used
type: CastDestinationType; // type of cast destination
isAvailable: boolean; // can we use the cast destination
isConnected: boolean; // is the cast destination actively sharing
currentTime: number | null; // current seek time the player is at
duration: number | null; // duration of media
receiverName: string | null; // name of the cast destination
castState: CastState; // current state of the cast destination
loadMedia(mediaUrl: string, sessionKey: string, reload: boolean): Promise<void>; // load media to the cast destination
// remote player controls
play(): void;
pause(): void;
seekTo(time: number): void;
disconnect(): void;
}
class CastManager {
private castDestinations = $state<ICastDestination[]>([]);
private current = $derived<ICastDestination | null>(this.monitorConnectedDestination());
availableDestinations = $state<ICastDestination[]>([]);
initialized = $state(false);
isCasting = $derived<boolean>(this.current?.isConnected ?? false);
receiverName = $derived<string | null>(this.current?.receiverName ?? null);
castState = $derived<CastState | null>(this.current?.castState ?? null);
currentTime = $derived<number | null>(this.current?.currentTime ?? null);
duration = $derived<number | null>(this.current?.duration ?? null);
private sessionKey: SessionCreateResponseDto | null = null;
constructor() {
// load each cast destination
this.castDestinations = [
new GCastDestination(),
// Add other cast destinations here (ie FCast)
];
}
async initialize() {
// this goes first to prevent multiple calls to initialize
if (this.initialized) {
return;
}
this.initialized = true;
// try to initialize each cast destination
for (const castDestination of this.castDestinations) {
const destAvailable = await castDestination.initialize();
if (destAvailable) {
this.availableDestinations.push(castDestination);
}
}
}
// monitor all cast destinations for changes
// we want to make sure only one session is active at a time
private monitorConnectedDestination(): ICastDestination | null {
// check if we have a connected destination
const connectedDest = this.castDestinations.find((dest) => dest.isConnected);
return connectedDest || null;
}
private isTokenValid() {
// check if we already have a session token
// we should always have a expiration date
if (!this.sessionKey || !this.sessionKey.expiresAt) {
return false;
}
const tokenExpiration = DateTime.fromISO(this.sessionKey.expiresAt);
// we want to make sure we have at least 10 seconds remaining in the session
// this is to account for network latency and other delays when sending the request
const bufferedExpiration = tokenExpiration.minus({ seconds: 10 });
return bufferedExpiration > DateTime.now();
}
private async refreshSessionToken() {
// get session token to authenticate the media url
// check and make sure we have at least 10 seconds remaining in the session
// before we send the media request, refresh the session if needed
if (!this.isTokenValid()) {
this.sessionKey = await createSession({
sessionCreateDto: {
duration: Duration.fromObject({ minutes: 15 }).as('seconds'),
deviceOS: 'Google Cast',
deviceType: 'Cast',
},
});
}
}
async loadMedia(mediaUrl: string, reload: boolean = false) {
if (!this.current) {
throw new Error('No active cast destination');
}
await this.refreshSessionToken();
if (!this.sessionKey) {
throw new Error('No session key available');
}
await this.current.loadMedia(mediaUrl, this.sessionKey.token, reload);
}
play() {
this.current?.play();
}
pause() {
this.current?.pause();
}
seekTo(time: number) {
this.current?.seekTo(time);
}
disconnect() {
this.current?.disconnect();
}
}
// Persist castManager across Svelte HMRs
let castManager: CastManager;
if (import.meta.hot && import.meta.hot.data) {
if (!import.meta.hot.data.castManager) {
import.meta.hot.data.castManager = new CastManager();
}
castManager = import.meta.hot.data.castManager;
} else {
castManager = new CastManager();
}
export { castManager };

View File

@@ -0,0 +1,234 @@
import { CastDestinationType, CastState, type ICastDestination } from '$lib/managers/cast-manager.svelte';
import 'chromecast-caf-sender';
import { Duration } from 'luxon';
const FRAMEWORK_LINK = 'https://www.gstatic.com/cv/js/sender/v1/cast_sender.js?loadCastFramework=1';
enum SESSION_DISCOVERY_CAUSE {
LOAD_MEDIA,
ACTIVE_SESSION,
}
export class GCastDestination implements ICastDestination {
type = CastDestinationType.GCAST;
isAvailable = $state<boolean>(false);
isConnected = $state<boolean>(false);
currentTime = $state<number | null>(null);
duration = $state<number | null>(null);
castState = $state<CastState>(CastState.IDLE);
receiverName = $state<string | null>(null);
private remotePlayer: cast.framework.RemotePlayer | null = null;
private session: chrome.cast.Session | null = null;
private currentMedia: chrome.cast.media.Media | null = null;
private currentUrl: string | null = null;
async initialize(): Promise<boolean> {
// this is a really messy way since google does a pseudo-callbak
// in the form of a global window event. We will give Chrome 3 seconds to respond
// or we will mark the destination as unavailable
const callbackPromise: Promise<boolean> = new Promise((resolve) => {
// check if the cast framework is already loaded
if (this.isAvailable) {
resolve(true);
return;
}
window['__onGCastApiAvailable'] = (isAvailable: boolean) => {
resolve(isAvailable);
};
if (!document.querySelector(`script[src="${FRAMEWORK_LINK}"]`)) {
const script = document.createElement('script');
script.src = FRAMEWORK_LINK;
document.body.append(script);
}
});
const timeoutPromise: Promise<boolean> = new Promise((resolve) => {
setTimeout(
() => {
resolve(false);
},
Duration.fromObject({ seconds: 3 }).toMillis(),
);
});
this.isAvailable = await Promise.race([callbackPromise, timeoutPromise]);
if (!this.isAvailable) {
return false;
}
const castContext = cast.framework.CastContext.getInstance();
this.remotePlayer = new cast.framework.RemotePlayer();
castContext.setOptions({
receiverApplicationId: chrome.cast.media.DEFAULT_MEDIA_RECEIVER_APP_ID,
autoJoinPolicy: chrome.cast.AutoJoinPolicy.ORIGIN_SCOPED,
});
castContext.addEventListener(cast.framework.CastContextEventType.SESSION_STATE_CHANGED, (event) =>
this.onSessionStateChanged(event),
);
castContext.addEventListener(cast.framework.CastContextEventType.CAST_STATE_CHANGED, (event) =>
this.onCastStateChanged(event),
);
const remotePlayerController = new cast.framework.RemotePlayerController(this.remotePlayer);
remotePlayerController.addEventListener(cast.framework.RemotePlayerEventType.ANY_CHANGE, (event) =>
this.onRemotePlayerChange(event),
);
return true;
}
async loadMedia(mediaUrl: string, sessionKey: string, reload: boolean = false): Promise<void> {
if (!this.isAvailable || !this.isConnected || !this.session) {
return;
}
// already playing the same media
if (this.currentUrl === mediaUrl && !reload) {
return;
}
// we need to send content type in the request
// in the future we can swap this out for an API call to get image metadata
const assetHead = await fetch(mediaUrl, { method: 'HEAD' });
const contentType = assetHead.headers.get('content-type');
if (!contentType) {
throw new Error('No content type found for media url');
}
// build the authenticated media request and send it to the cast device
const authenticatedUrl = `${mediaUrl}&sessionKey=${sessionKey}`;
const mediaInfo = new chrome.cast.media.MediaInfo(authenticatedUrl, contentType);
const request = new chrome.cast.media.LoadRequest(mediaInfo);
const successCallback = this.onMediaDiscovered.bind(this, SESSION_DISCOVERY_CAUSE.LOAD_MEDIA);
this.currentUrl = mediaUrl;
return this.session.loadMedia(request, successCallback, this.onError.bind(this));
}
///
/// Remote Player Controls
///
play(): void {
if (!this.currentMedia) {
return;
}
const playRequest = new chrome.cast.media.PlayRequest();
this.currentMedia.play(playRequest, () => {}, this.onError.bind(this));
}
pause(): void {
if (!this.currentMedia) {
return;
}
const pauseRequest = new chrome.cast.media.PauseRequest();
this.currentMedia.pause(pauseRequest, () => {}, this.onError.bind(this));
}
seekTo(time: number): void {
const remotePlayer = new cast.framework.RemotePlayer();
const remotePlayerController = new cast.framework.RemotePlayerController(remotePlayer);
remotePlayer.currentTime = time;
remotePlayerController.seek();
}
disconnect(): void {
this.session?.leave(() => {
this.session = null;
this.castState = CastState.IDLE;
this.isConnected = false;
this.receiverName = null;
}, this.onError.bind(this));
}
///
/// Google Cast Callbacks
///
private onSessionStateChanged(event: cast.framework.SessionStateEventData) {
switch (event.sessionState) {
case cast.framework.SessionState.SESSION_ENDED: {
this.session = null;
break;
}
case cast.framework.SessionState.SESSION_RESUMED:
case cast.framework.SessionState.SESSION_STARTED: {
this.session = event.session.getSessionObj();
break;
}
}
}
private onCastStateChanged(event: cast.framework.CastStateEventData) {
this.isConnected = event.castState === cast.framework.CastState.CONNECTED;
this.receiverName = this.session?.receiver.friendlyName ?? null;
if (event.castState === cast.framework.CastState.NOT_CONNECTED) {
this.currentMedia = null;
this.currentUrl = null;
}
}
private onRemotePlayerChange(event: cast.framework.RemotePlayerChangedEvent) {
switch (event.field) {
case 'isConnected': {
this.isConnected = event.value;
break;
}
case 'remotePlayer': {
this.remotePlayer = event.value;
break;
}
case 'duration': {
this.duration = event.value;
break;
}
case 'currentTime': {
this.currentTime = event.value;
break;
}
case 'playerState': {
this.castState = event.value;
break;
}
}
}
onError(error: chrome.cast.Error) {
console.error('Google Cast Error:', error);
}
private onMediaDiscovered(cause: SESSION_DISCOVERY_CAUSE, currentMedia: chrome.cast.media.Media) {
this.currentMedia = currentMedia;
if (cause === SESSION_DISCOVERY_CAUSE.LOAD_MEDIA) {
this.castState = CastState.PLAYING;
} else if (cause === SESSION_DISCOVERY_CAUSE.ACTIVE_SESSION) {
// CastState and PlayerState are identical enums
this.castState = currentMedia.playerState as unknown as CastState;
}
}
static async showCastDialog() {
try {
await cast.framework.CastContext.getInstance().requestSession();
} catch {
// the cast dialog throws an error if the user closes it
// we don't care about this error
return;
}
}
}

View File

@@ -1,6 +1,7 @@
<script lang="ts">
import { afterNavigate, goto, onNavigate } from '$app/navigation';
import { scrollMemoryClearer } from '$lib/actions/scroll-memory';
import CastButton from '$lib/cast/cast-button.svelte';
import AlbumDescription from '$lib/components/album-page/album-description.svelte';
import AlbumMap from '$lib/components/album-page/album-map.svelte';
import AlbumOptions from '$lib/components/album-page/album-options.svelte';
@@ -620,6 +621,8 @@
{#if viewMode === AlbumPageViewMode.VIEW}
<ControlAppBar showBackButton backIcon={mdiArrowLeft} onClose={() => goto(backUrl)}>
{#snippet trailing()}
<CastButton whiteHover />
{#if isEditor}
<CircleIconButton
title={$t('add_photos')}