✨(frontend) add adaptive PiP stage with focus and grid layouts

Render all participants with a focus mode and animated adaptive grid.

src/frontend/src/features/pip/components/PipView.tsx

@@ -1,91 +1,20 @@
-import { styled } from '@/styled-system/jsx'
 import { supportsScreenSharing } from '@livekit/components-core'
-import {
+import { styled } from '@/styled-system/jsx'
-  isTrackReference,
-  TrackReferenceOrPlaceholder,
-} from '@livekit/components-core'
-import { useTracks } from '@livekit/components-react'
-import { Track } from 'livekit-client'
-import { ParticipantTile } from '@/features/rooms/livekit/components/ParticipantTile'
 import { SidePanel } from '@/features/rooms/livekit/components/SidePanel'
 import { pipLayoutStore } from '../stores/pipLayoutStore'
 import { PipControlBar } from './PipControlBar'
 import { PipReactionsToolbar } from './PipReactionsToolbar'
+import { PipStage } from './layouts/PipStage'
 
-const pickTrackForPip = (
-  tracks: TrackReferenceOrPlaceholder[]
-): TrackReferenceOrPlaceholder | undefined => {
-  // Prefer screen share when present; otherwise fallback to first available track.
-  const screenShareTrack = tracks
-    .filter((track) => isTrackReference(track))
-    .find((track) => track.publication.source === Track.Source.ScreenShare)
 
-  if (screenShareTrack) return screenShareTrack
+// Composition shell for the Picture-in-Picture window.
-  return tracks[0]
+ 
-}
-
-const pickLocalCameraTrack = (
-  tracks: TrackReferenceOrPlaceholder[]
-): TrackReferenceOrPlaceholder | undefined =>
-  tracks.find(
-    (track) =>
-      track.source === Track.Source.Camera && track.participant?.isLocal
-  )
-
-const pickRemoteCameraTrack = (
-  tracks: TrackReferenceOrPlaceholder[]
-): TrackReferenceOrPlaceholder | undefined =>
-  tracks.find(
-    (track) =>
-      track.source === Track.Source.Camera && !track.participant?.isLocal
-  )
-
-/**
- * Main view component for the Picture-in-Picture window.
- * Handles track selection (prioritizes screen share), layout switching (grid for multiple participants),
- * and renders the control bar and side panel within the PiP window.
- */
 export const PipView = () => {
-  const tracks = useTracks(
-    [
-      { source: Track.Source.Camera, withPlaceholder: true },
-      { source: Track.Source.ScreenShare, withPlaceholder: false },
-    ],
-    { onlySubscribed: false }
-  )
-
-  const trackRef = pickTrackForPip(tracks)
   const browserSupportsScreenSharing = supportsScreenSharing()
-  const hasMultipleTiles = tracks.length > 1
-  const localCameraTrack = pickLocalCameraTrack(tracks)
-  const remoteCameraTrack = pickRemoteCameraTrack(tracks)
-  const mainTrackRef = remoteCameraTrack ?? trackRef
-  const thumbnailTrackRef =
-    mainTrackRef && localCameraTrack && localCameraTrack !== mainTrackRef
-      ? localCameraTrack
-      : tracks.find((track) => track !== mainTrackRef)
-
-  if (!trackRef && !hasMultipleTiles) return null
 
   return (
     <PipContainer>
-      {/* Keep stage height stable to avoid layout shifting on track changes. */}
+      <PipStage />
-      <PipStage>
-        {hasMultipleTiles ? (
-          <>
-            {mainTrackRef && (
-              <ParticipantTile trackRef={mainTrackRef} disableMetadata />
-            )}
-            {thumbnailTrackRef && (
-              <PipThumbnail>
-                <ParticipantTile trackRef={thumbnailTrackRef} disableMetadata />
-              </PipThumbnail>
-            )}
-          </>
-        ) : (
-          <ParticipantTile trackRef={trackRef} disableMetadata />
-        )}
-      </PipStage>
       <PipReactionsToolbar />
       <PipControlBar showScreenShare={browserSupportsScreenSharing} />
       {/* Side panel (effects, settings, etc.) opens within PiP window. */}
@@ -117,29 +46,3 @@ const PipContainer = styled('div', {
     },
   },
 })
-
-const PipStage = styled('div', {
-  base: {
-    position: 'relative',
-    minHeight: 0,
-    margin: '0.5rem',
-    borderRadius: 'lg',
-    overflow: 'hidden',
-  },
-})
-
-const PipThumbnail = styled('div', {
-  base: {
-    position: 'absolute',
-    right: '1rem',
-    bottom: '1rem',
-    width: '42%',
-    maxWidth: '220px',
-    minWidth: '140px',
-    aspectRatio: '16 / 9',
-    borderRadius: 'md',
-    overflow: 'hidden',
-    boxShadow: 'md',
-    zIndex: 2,
-  },
-})

src/frontend/src/features/pip/components/layouts/PipFocusLayout.tsx

@@ -0,0 +1,79 @@
+import { memo } from 'react'
+import type { TrackReferenceOrPlaceholder } from '@livekit/components-core'
+import { styled } from '@/styled-system/jsx'
+import { ParticipantTile } from '@/features/rooms/livekit/components/ParticipantTile'
+import { getTrackKey } from '../../utils/pipTrackSelection'
+
+type PipFocusLayoutProps = {
+  mainTrack: TrackReferenceOrPlaceholder
+  thumbnailTrack?: TrackReferenceOrPlaceholder
+}
+
+/**
+ * Focus layout used when 1-2 tracks are visible in the PiP window.
+ *
+ * The main tile is letterboxed (object-fit: contain) so the camera is
+ * never stretched to a non-video aspect and leaves dark padding
+ * above/below when the window shape doesn't match the source.
+ * The thumbnail keeps the usual cover fill.
+ */
+export const PipFocusLayout = memo(function PipFocusLayout({
+  mainTrack,
+  thumbnailTrack,
+}: PipFocusLayoutProps) {
+  return (
+    <FocusContainer>
+      <MainSlot>
+        <ParticipantTile
+          key={getTrackKey(mainTrack)}
+          trackRef={mainTrack}
+          disableMetadata
+        />
+      </MainSlot>
+      {thumbnailTrack && (
+        <Thumbnail>
+          <ParticipantTile
+            key={getTrackKey(thumbnailTrack)}
+            trackRef={thumbnailTrack}
+            disableMetadata
+          />
+        </Thumbnail>
+      )}
+    </FocusContainer>
+  )
+})
+
+const FocusContainer = styled('div', {
+  base: {
+    position: 'relative',
+    width: '100%',
+    height: '100%',
+    backgroundColor: 'primaryDark.100',
+  },
+})
+
+const MainSlot = styled('div', {
+  base: {
+    width: '100%',
+    height: '100%',
+    '& .lk-participant-media-video': {
+      objectFit: 'contain',
+    },
+  },
+})
+
+const Thumbnail = styled('div', {
+  base: {
+    position: 'absolute',
+    right: '1rem',
+    bottom: '1rem',
+    width: '42%',
+    maxWidth: '220px',
+    minWidth: '140px',
+    aspectRatio: '16 / 9',
+    borderRadius: 'md',
+    overflow: 'hidden',
+    boxShadow: 'md',
+    zIndex: 2,
+  },
+})

src/frontend/src/features/pip/components/layouts/PipGridLayout.tsx

@@ -0,0 +1,85 @@
+import { memo, useMemo, useRef } from 'react'
+import type { TrackReferenceOrPlaceholder } from '@livekit/components-core'
+import { styled } from '@/styled-system/jsx'
+import { ParticipantTile } from '@/features/rooms/livekit/components/ParticipantTile'
+import { usePipElementSize } from '../../hooks/usePipElementSize'
+import { usePipFlipAnimations } from '../../hooks/usePipFlipAnimations'
+import { computePipGridLayout } from '../../utils/pipGrid'
+import { getTrackKey } from '../../utils/pipTrackSelection'
+
+type PipGridLayoutProps = {
+  tracks: TrackReferenceOrPlaceholder[]
+}
+
+/**
+ * Adaptive grid used when 3+ tracks are visible in the PiP window.
+ *
+ * All grid math (shape choice + partial-row stretching) is delegated to
+ * `computePipGridLayout`. This component only measures the container,
+ * applies the returned placements, and plays a FLIP animation when the
+ * tile set or grid shape changes (participant joins/leaves or shape shift).
+ *
+ * Tiles keep a stable key so resizing never remounts <video> elements.
+ */
+export const PipGridLayout = memo(function PipGridLayout({
+  tracks,
+}: PipGridLayoutProps) {
+  const containerRef = useRef<HTMLDivElement>(null)
+  const { width, height } = usePipElementSize(containerRef)
+
+  const tileKeys = useMemo(() => tracks.map(getTrackKey), [tracks])
+
+  const { rows, subColumns, placements } = useMemo(
+    () => computePipGridLayout(tracks.length, width, height),
+    [tracks.length, width, height]
+  )
+
+  const gridStyle = useMemo(
+    () => ({
+      gridTemplateColumns: `repeat(${subColumns}, minmax(0, 1fr))`,
+      gridTemplateRows: `repeat(${rows}, minmax(0, 1fr))`,
+    }),
+    [subColumns, rows]
+  )
+
+  usePipFlipAnimations(containerRef, tileKeys)
+
+  return (
+    <GridContainer ref={containerRef} style={gridStyle}>
+      {tracks.map((track, index) => (
+        <GridCell
+          key={tileKeys[index]}
+          style={placements[index]}
+        >
+          <ParticipantTile trackRef={track} disableMetadata />
+        </GridCell>
+      ))}
+    </GridContainer>
+  )
+})
+
+const GridContainer = styled('div', {
+  base: {
+    width: '100%',
+    height: '100%',
+    display: 'grid',
+    gap: '0.25rem',
+  },
+})
+
+const GridCell = styled('div', {
+  base: {
+    position: 'relative',
+    minWidth: 0,
+    minHeight: 0,
+    borderRadius: 'md',
+    overflow: 'hidden',
+    backgroundColor: 'primaryDark.100',
+    // Paint on own layer so FLIP transforms don't trigger layout thrash.
+    willChange: 'transform',
+    '& .lk-participant-tile': {
+      width: '100%',
+      height: '100%',
+    },
+  },
+})

src/frontend/src/features/pip/components/layouts/PipStage.tsx

@@ -0,0 +1,78 @@
+import { useMemo } from 'react'
+import { useTracks } from '@livekit/components-react'
+import { Track } from 'livekit-client'
+import { styled } from '@/styled-system/jsx'
+import {
+  isCameraTrack,
+  pickLocalCameraTrack,
+  pickRemoteCameraTrack,
+  pickScreenShareTrack,
+} from '../../utils/pipTrackSelection'
+import { PipFocusLayout } from './PipFocusLayout'
+import { PipGridLayout } from './PipGridLayout'
+
+/**
+ * Above this count the PiP stage switches from the focus layout
+ * (main + thumbnail) to the adaptive grid layout.
+ */
+const FOCUS_MAX_TILES = 2
+ 
+// Handles which layout to render inside the PiP stage.
+
+export const PipStage = () => {
+  const tracks = useTracks(
+    [
+      { source: Track.Source.Camera, withPlaceholder: true },
+      { source: Track.Source.ScreenShare, withPlaceholder: false },
+    ],
+    { onlySubscribed: false }
+  )
+
+  const screenShareTrack = useMemo(
+    () => pickScreenShareTrack(tracks),
+    [tracks]
+  )
+
+  // Order the list so the "focus target" (screen share when available,
+  // otherwise a remote camera) is first. Both layouts consume this order.
+  const stageTracks = useMemo(() => {
+    const cameraTracks = tracks.filter(isCameraTrack)
+    if (!screenShareTrack) return cameraTracks
+    return [screenShareTrack, ...cameraTracks]
+  }, [tracks, screenShareTrack])
+
+  if (stageTracks.length === 0) return null
+
+  if (stageTracks.length > FOCUS_MAX_TILES) {
+    return (
+      <StageFrame>
+        <PipGridLayout tracks={stageTracks} />
+      </StageFrame>
+    )
+  }
+
+  const localCameraTrack = pickLocalCameraTrack(stageTracks)
+  const remoteCameraTrack = pickRemoteCameraTrack(stageTracks)
+  const mainTrack = screenShareTrack ?? remoteCameraTrack ?? stageTracks[0]
+  const thumbnailTrack =
+    localCameraTrack && localCameraTrack !== mainTrack
+      ? localCameraTrack
+      : stageTracks.find((track) => track !== mainTrack)
+
+  return (
+    <StageFrame>
+      <PipFocusLayout mainTrack={mainTrack} thumbnailTrack={thumbnailTrack} />
+    </StageFrame>
+  )
+}
+
+const StageFrame = styled('div', {
+  base: {
+    position: 'relative',
+    minWidth: 0,
+    minHeight: 0,
+    margin: '0.5rem',
+    borderRadius: 'lg',
+    overflow: 'hidden',
+  },
+})

src/frontend/src/features/pip/hooks/usePipFlipAnimations.ts

@@ -0,0 +1,90 @@
+import { useLayoutEffect, useRef, type RefObject } from 'react'
+
+type Options = {
+  /** Animation duration in ms. */
+  duration?: number
+  /** CSS easing function. */
+  easing?: string
+}
+
+/**
+ * FLIP (First, Last, Invert, Play) animation hook.
+ *
+ * For every keyed direct child of `containerRef`, records its position
+ * before a render (the "first" rect) and, once the DOM has committed, plays
+ * an inverse transform back to the identity position. The effect is a
+ * smooth slide whenever tiles are added, removed, reordered, or a new grid
+ * shape shifts them.
+ *
+ * Safe to call inside a Document PiP window: uses the element's own
+ * Web Animations API (element.animate) which lives in the PiP document.
+ * Respects `prefers-reduced-motion` and no-ops on the first mount.
+ */
+export const usePipFlipAnimations = <T extends HTMLElement>(
+  containerRef: RefObject<T | null>,
+  keys: ReadonlyArray<string>,
+  { duration = 220, easing = 'cubic-bezier(0.2, 0, 0, 1)' }: Options = {}
+) => {
+  const prevRectsRef = useRef<Map<string, DOMRect>>(new Map())
+  const firstRunRef = useRef(true)
+
+  useLayoutEffect(() => {
+    const container = containerRef.current
+    if (!container) return
+
+    const doc = container.ownerDocument
+    const view = doc.defaultView
+    const reduceMotion = view?.matchMedia('(prefers-reduced-motion: reduce)')
+      .matches
+
+    const children = Array.from(container.children) as HTMLElement[]
+    const nextRects = new Map<string, DOMRect>()
+    children.forEach((el, i) => {
+      const key = keys[i]
+      if (!key) return
+      nextRects.set(key, el.getBoundingClientRect())
+    })
+
+    if (firstRunRef.current) {
+      firstRunRef.current = false
+      prevRectsRef.current = nextRects
+      return
+    }
+
+    if (!reduceMotion) {
+      children.forEach((el, i) => {
+        const key = keys[i]
+        if (!key) return
+        const prev = prevRectsRef.current.get(key)
+        const next = nextRects.get(key)
+        if (!prev || !next) return
+
+        const dx = prev.left - next.left
+        const dy = prev.top - next.top
+        const sx = next.width === 0 ? 1 : prev.width / next.width
+        const sy = next.height === 0 ? 1 : prev.height / next.height
+
+        // Skip no-ops: sub-pixel shifts don't benefit from animation.
+        if (
+          Math.abs(dx) < 1 &&
+          Math.abs(dy) < 1 &&
+          Math.abs(sx - 1) < 0.01 &&
+          Math.abs(sy - 1) < 0.01
+        )
+          return
+
+        el.animate(
+          [
+            {
+              transform: `translate(${dx}px, ${dy}px) scale(${sx}, ${sy})`,
+            },
+            { transform: 'translate(0, 0) scale(1, 1)' },
+          ],
+          { duration, easing, fill: 'backwards' }
+        )
+      })
+    }
+
+    prevRectsRef.current = nextRects
+  }, [containerRef, duration, easing, keys])
+}

src/frontend/src/features/pip/utils/pipGrid.ts

@@ -0,0 +1,110 @@
+export type PipTilePlacement = {
+  gridColumn: string
+  gridRow: number
+}
+
+export type PipGridLayout = {
+  cols: number
+  rows: number
+  /** Number of CSS sub-columns; use as `repeat(subColumns, 1fr)`. */
+  subColumns: number
+  /** One entry per tile, in input order. */
+  placements: PipTilePlacement[]
+}
+
+/**
+ * Target tile aspect ratio used to score candidate grid shapes.
+ *
+ * Video sources are 16:9, but picking 16:9 as the target makes the
+ * scorer indifferent between a stretched 2-col slab (aspect ~2.7) and a
+ * squarer 3-col tile (aspect ~1.2) because log distance is symmetric.
+ * The UI works better with square, face-friendly tiles. This target keeps
+ * wide windows from collapsing to 2 columns with short, stretched rows
+ * and pushes the scorer to add a column instead.
+ */
+const TARGET_TILE_ASPECT = 1
+
+/**
+ * Smallest count from which we force at least two columns.
+ * For 1-3 participants it is acceptable to stack vertically in tall
+ * windows, but from 4 people onwards we keep >=2 columns to
+ * avoid endless vertical scrolling; the scorer handles the rest.
+ */
+const FORCE_TWO_COLS_COUNT = 4
+
+const pickGridShape = (
+  count: number,
+  width: number,
+  height: number
+): { cols: number; rows: number } => {
+  if (count <= 1) return { cols: 1, rows: Math.max(1, count) }
+  if (width <= 0 || height <= 0) return { cols: count, rows: 1 }
+
+  const minCols = count >= FORCE_TWO_COLS_COUNT ? 2 : 1
+
+  let best = { cols: minCols, rows: Math.ceil(count / minCols), score: -Infinity }
+  for (let cols = minCols; cols <= count; cols++) {
+    const rows = Math.ceil(count / cols)
+    const tileW = width / cols
+    const tileH = height / rows
+    if (tileW <= 0 || tileH <= 0) continue
+
+    // Score: aspect close to target, few empty cells, large tile area,
+    // and a tiny bias toward fewer rows so ties (perfectly square shapes)
+    // resolve in favour of a shorter, wider grid.
+    const aspectScore = -Math.abs(Math.log(tileW / tileH / TARGET_TILE_ASPECT))
+    const emptyCells = cols * rows - count
+    const fillScore = -emptyCells * 0.1
+    const areaScore = Math.log(tileW * tileH) * 0.5
+    const rowsPenalty = -rows * 0.01
+
+    const score = aspectScore * 2 + fillScore + areaScore + rowsPenalty
+    if (score > best.score) best = { cols, rows, score }
+  }
+  return { cols: best.cols, rows: best.rows }
+}
+
+/**
+ * Pure function. Given a tile count and stage dimensions, returns the CSS
+ * grid layout for the PiP stage:
+ *
+ *  - picks a cols x rows shape close to 16:9 tiles,
+ *  - stretches any partial last row so its tiles share the full row width
+ *    (no empty cells, no small centered tile).
+ *
+ * Callers consume the result directly: `subColumns` feeds
+ * `grid-template-columns: repeat(N, 1fr)` and each tile reads its own
+ * `gridColumn`/`gridRow` from `placements`.
+ */
+export const computePipGridLayout = (
+  count: number,
+  width: number,
+  height: number
+): PipGridLayout => {
+  if (count <= 0) {
+    return { cols: 1, rows: 1, subColumns: 1, placements: [] }
+  }
+
+  const { cols, rows } = pickGridShape(count, width, height)
+  const tilesInLastRow = count - cols * (rows - 1)
+  const hasPartialRow = tilesInLastRow > 0 && tilesInLastRow < cols
+
+  const subColumns = hasPartialRow ? cols * tilesInLastRow : cols
+  const fullRowSpan = hasPartialRow ? tilesInLastRow : 1
+  const lastRowSpan = hasPartialRow ? cols : 1
+
+  const placements: PipTilePlacement[] = []
+  for (let i = 0; i < count; i++) {
+    const row = Math.floor(i / cols)
+    const colIndex = i % cols
+    const isLastRow = row === rows - 1 && hasPartialRow
+    const span = isLastRow ? lastRowSpan : fullRowSpan
+    const colStart = colIndex * span + 1
+    placements.push({
+      gridColumn: `${colStart} / span ${span}`,
+      gridRow: row + 1,
+    })
+  }
+
+  return { cols, rows, subColumns, placements }
+}

src/frontend/src/features/pip/utils/pipTrackSelection.ts

@@ -0,0 +1,48 @@
+import {
+  isTrackReference,
+  TrackReferenceOrPlaceholder,
+} from '@livekit/components-core'
+import { Track } from 'livekit-client'
+
+/**
+ * Helpers used by the PiP layouts to classify/pick tracks.
+ * Kept free of React so they are trivially testable and cheap to call.
+ */
+
+export const pickScreenShareTrack = (
+  tracks: TrackReferenceOrPlaceholder[]
+): TrackReferenceOrPlaceholder | undefined =>
+  tracks
+    .filter((track) => isTrackReference(track))
+    .find((track) => track.publication.source === Track.Source.ScreenShare)
+
+export const pickLocalCameraTrack = (
+  tracks: TrackReferenceOrPlaceholder[]
+): TrackReferenceOrPlaceholder | undefined =>
+  tracks.find(
+    (track) =>
+      track.source === Track.Source.Camera && track.participant?.isLocal
+  )
+
+export const pickRemoteCameraTrack = (
+  tracks: TrackReferenceOrPlaceholder[]
+): TrackReferenceOrPlaceholder | undefined =>
+  tracks.find(
+    (track) =>
+      track.source === Track.Source.Camera && !track.participant?.isLocal
+  )
+
+export const isCameraTrack = (track: TrackReferenceOrPlaceholder): boolean =>
+  track.source === Track.Source.Camera
+
+/**
+ * Produces a stable React key for a track so resizes/reshuffles of the grid
+ * do not remount the underlying <video> element.
+ */
+export const getTrackKey = (track: TrackReferenceOrPlaceholder): string => {
+  const identity = track.participant?.identity ?? 'unknown'
+  if (isTrackReference(track)) {
+    return `${identity}::${track.source}::${track.publication.trackSid}`
+  }
+  return `${identity}::${track.source}::placeholder`
+}