Better support for PCM audio formats around DefaultAudioSink

This change:
- abstracts away the fact that SonicAudioProcessor does not support float PCM
  from the audio sink to the audio processor chain (easier to replace)
- makes the audio sink allow custom audio processors with any output format
- adds DSD and double PCM format definitions to allow passing these around
  for internal signal processing
- adds a new mode to DefaultAudioSink that allows any supported linear PCM format
  to be output into the AudioTrack

Issue: google/ExoPlayer#10516
Issue: #1931

Author: nift4

libraries/common/src/main/java/androidx/media3/common/C.java

@@ -168,10 +168,10 @@ private C() {}
    * {@link #ENCODING_INVALID}, {@link #ENCODING_PCM_8BIT}, {@link #ENCODING_PCM_16BIT}, {@link
    * #ENCODING_PCM_16BIT_BIG_ENDIAN}, {@link #ENCODING_PCM_24BIT}, {@link
    * #ENCODING_PCM_24BIT_BIG_ENDIAN}, {@link #ENCODING_PCM_32BIT}, {@link
-   * #ENCODING_PCM_32BIT_BIG_ENDIAN}, {@link #ENCODING_PCM_FLOAT}, {@link #ENCODING_MP3}, {@link
-   * #ENCODING_AC3}, {@link #ENCODING_E_AC3}, {@link #ENCODING_E_AC3_JOC}, {@link #ENCODING_AC4},
-   * {@link #ENCODING_DTS}, {@link #ENCODING_DTS_HD}, {@link #ENCODING_DOLBY_TRUEHD} or {@link
-   * #ENCODING_OPUS}.
+   * #ENCODING_PCM_32BIT_BIG_ENDIAN}, {@link #ENCODING_PCM_FLOAT}, {@link #ENCODING_PCM_DOUBLE},
+   * {@link #ENCODING_MP3}, {@link #ENCODING_AC3}, {@link #ENCODING_E_AC3}, {@link
+   * #ENCODING_E_AC3_JOC}, {@link #ENCODING_AC4}, {@link #ENCODING_DTS}, {@link #ENCODING_DTS_HD},
+   * {@link #ENCODING_DOLBY_TRUEHD}, {@link #ENCODING_OPUS} or {@link #ENCODING_DSD}.
    */
   @UnstableApi
   @Documented
@@ -188,6 +188,7 @@ private C() {}
     ENCODING_PCM_32BIT,
     ENCODING_PCM_32BIT_BIG_ENDIAN,
     ENCODING_PCM_FLOAT,
+    ENCODING_PCM_DOUBLE,
     ENCODING_MP3,
     ENCODING_AAC_LC,
     ENCODING_AAC_HE_V1,
@@ -204,6 +205,7 @@ private C() {}
     ENCODING_DOLBY_TRUEHD,
     ENCODING_OPUS,
     ENCODING_DTS_UHD_P2,
+    ENCODING_DSD,
   })
   public @interface Encoding {}
 
@@ -212,7 +214,7 @@ private C() {}
    * {@link #ENCODING_INVALID}, {@link #ENCODING_PCM_8BIT}, {@link #ENCODING_PCM_16BIT}, {@link
    * #ENCODING_PCM_16BIT_BIG_ENDIAN}, {@link #ENCODING_PCM_24BIT}, {@link
    * #ENCODING_PCM_24BIT_BIG_ENDIAN}, {@link #ENCODING_PCM_32BIT}, {@link
-   * #ENCODING_PCM_32BIT_BIG_ENDIAN}, {@link #ENCODING_PCM_FLOAT}.
+   * #ENCODING_PCM_32BIT_BIG_ENDIAN}, {@link #ENCODING_PCM_FLOAT}, {@link #ENCODING_PCM_DOUBLE}.
    */
   @UnstableApi
   @Documented
@@ -228,7 +230,8 @@ private C() {}
     ENCODING_PCM_24BIT_BIG_ENDIAN,
     ENCODING_PCM_32BIT,
     ENCODING_PCM_32BIT_BIG_ENDIAN,
-    ENCODING_PCM_FLOAT
+    ENCODING_PCM_FLOAT,
+    ENCODING_PCM_DOUBLE
   })
   public @interface PcmEncoding {}
 
@@ -259,6 +262,9 @@ private C() {}
   /** See {@link AudioFormat#ENCODING_PCM_FLOAT}. */
   @UnstableApi public static final int ENCODING_PCM_FLOAT = AudioFormat.ENCODING_PCM_FLOAT;
 
+  /** PCM encoding with double-precision floating point samples. */
+  @UnstableApi public static final int ENCODING_PCM_DOUBLE = 0x70000000;
+
   /** See {@link AudioFormat#ENCODING_MP3}. */
   @UnstableApi public static final int ENCODING_MP3 = AudioFormat.ENCODING_MP3;
 
@@ -307,6 +313,9 @@ private C() {}
   /** See {@link AudioFormat#ENCODING_OPUS}. */
   @UnstableApi public static final int ENCODING_OPUS = AudioFormat.ENCODING_OPUS;
 
+  /** See {@link AudioFormat#ENCODING_DSD}. */
+  @UnstableApi public static final int ENCODING_DSD = AudioFormat.ENCODING_DSD;
+
   /**
    * Represents the behavior affecting whether spatialization will be used. One of {@link
    * #SPATIALIZATION_BEHAVIOR_AUTO} or {@link #SPATIALIZATION_BEHAVIOR_NEVER}.

libraries/common/src/main/java/androidx/media3/common/MimeTypes.java

@@ -86,6 +86,7 @@ public final class MimeTypes {
   public static final String AUDIO_E_AC3_JOC = BASE_TYPE_AUDIO + "/eac3-joc";
   public static final String AUDIO_AC4 = BASE_TYPE_AUDIO + "/ac4";
   public static final String AUDIO_TRUEHD = BASE_TYPE_AUDIO + "/true-hd";
+  public static final String AUDIO_DSD = BASE_TYPE_AUDIO + "/x-dsd";
   public static final String AUDIO_DTS = BASE_TYPE_AUDIO + "/vnd.dts";
   public static final String AUDIO_DTS_HD = BASE_TYPE_AUDIO + "/vnd.dts.hd";
   public static final String AUDIO_DTS_EXPRESS = BASE_TYPE_AUDIO + "/vnd.dts.hd;profile=lbr";
@@ -686,6 +687,8 @@ public static boolean isDolbyVisionCodec(
         return C.ENCODING_DOLBY_TRUEHD;
       case MimeTypes.AUDIO_OPUS:
         return C.ENCODING_OPUS;
+      case MimeTypes.AUDIO_DSD:
+        return C.ENCODING_DSD;
       default:
         return C.ENCODING_INVALID;
     }

libraries/common/src/main/java/androidx/media3/common/audio/AudioProcessorChain.java

@@ -15,6 +15,7 @@
  */
 package androidx.media3.common.audio;
 
+import androidx.media3.common.Format;
 import androidx.media3.common.PlaybackParameters;
 import androidx.media3.common.util.UnstableApi;
 
@@ -33,7 +34,7 @@ public interface AudioProcessorChain {
    * during initialization, but audio processors may change state to become active/inactive during
    * playback.
    */
-  AudioProcessor[] getAudioProcessors();
+  AudioProcessor[] getAudioProcessors(Format inputFormat);
 
   /**
    * Configures audio processors to apply the specified playback parameters immediately, returning
@@ -50,7 +51,7 @@ public interface AudioProcessorChain {
    * value. Only called when processors have no input pending.
    *
    * @param skipSilenceEnabled Whether silences should be skipped in the audio stream.
-   * @return The new value.
+   * @return The value that was actually applied.
    */
   boolean applySkipSilenceEnabled(boolean skipSilenceEnabled);
 

libraries/common/src/main/java/androidx/media3/common/audio/ToInt16PcmAudioProcessor.java

@@ -34,6 +34,7 @@
  *   <li>{@link C#ENCODING_PCM_32BIT}
  *   <li>{@link C#ENCODING_PCM_32BIT_BIG_ENDIAN}
  *   <li>{@link C#ENCODING_PCM_FLOAT}
+ *   <li>{@link C#ENCODING_PCM_DOUBLE}
  * </ul>
  */
 @UnstableApi
@@ -50,7 +51,8 @@ public AudioFormat onConfigure(AudioFormat inputAudioFormat)
         && encoding != C.ENCODING_PCM_24BIT_BIG_ENDIAN
         && encoding != C.ENCODING_PCM_32BIT
         && encoding != C.ENCODING_PCM_32BIT_BIG_ENDIAN
-        && encoding != C.ENCODING_PCM_FLOAT) {
+        && encoding != C.ENCODING_PCM_FLOAT
+        && encoding != C.ENCODING_PCM_DOUBLE) {
       throw new UnhandledAudioFormatException(inputAudioFormat);
     }
     return encoding != C.ENCODING_PCM_16BIT
@@ -82,6 +84,9 @@ public void queueInput(ByteBuffer inputBuffer) {
       case C.ENCODING_PCM_FLOAT:
         resampledSize = size / 2;
         break;
+      case C.ENCODING_PCM_DOUBLE:
+        resampledSize = size / 4;
+        break;
       case C.ENCODING_PCM_16BIT:
       case C.ENCODING_INVALID:
       case Format.NO_VALUE:
@@ -147,6 +152,19 @@ public void queueInput(ByteBuffer inputBuffer) {
           buffer.put((byte) ((shortValue >> 8) & 0xFF));
         }
         break;
+      case C.ENCODING_PCM_DOUBLE:
+        // 64 bit floating point -> 16 bit resampling. Floating point values are in the range
+        // [-1.0, 1.0], so need to be scaled by Short.MAX_VALUE.
+        for (int i = position; i < limit; i += 8) {
+          // Clamp to avoid integer overflow if the floating point values exceed their nominal range
+          // [Internal ref: b/161204847].
+          double doubleValue =
+              Util.constrainValue(inputBuffer.getDouble(i), /* min= */ -1, /* max= */ 1);
+          short shortValue = (short) (doubleValue * Short.MAX_VALUE);
+          buffer.put((byte) (shortValue & 0xFF));
+          buffer.put((byte) ((shortValue >> 8) & 0xFF));
+        }
+        break;
       case C.ENCODING_PCM_16BIT:
       case C.ENCODING_INVALID:
       case Format.NO_VALUE:

libraries/common/src/main/java/androidx/media3/common/util/MediaFormatUtil.java

@@ -482,6 +482,7 @@ private static void maybeSetPcmEncoding(
       case C.ENCODING_PCM_16BIT_BIG_ENDIAN:
       case C.ENCODING_PCM_24BIT_BIG_ENDIAN:
       case C.ENCODING_PCM_32BIT_BIG_ENDIAN:
+      case C.ENCODING_PCM_DOUBLE:
       default:
         // No matching value. Do nothing.
         return;

libraries/common/src/main/java/androidx/media3/common/util/Util.java

@@ -1188,6 +1188,19 @@ public static float constrainValue(float value, float min, float max) {
     return max(min, min(value, max));
   }
 
+  /**
+   * Constrains a value to the specified bounds.
+   *
+   * @param value The value to constrain.
+   * @param min The lower bound.
+   * @param max The upper bound.
+   * @return The constrained value {@code Math.max(min, Math.min(value, max))}.
+   */
+  @UnstableApi
+  public static double constrainValue(double value, double min, double max) {
+    return max(min, min(value, max));
+  }
+
   /**
    * Returns the sum of two arguments, or a third argument if the result overflows.
    *
@@ -2258,7 +2271,8 @@ public static boolean isEncodingLinearPcm(@C.Encoding int encoding) {
         || encoding == C.ENCODING_PCM_24BIT_BIG_ENDIAN
         || encoding == C.ENCODING_PCM_32BIT
         || encoding == C.ENCODING_PCM_32BIT_BIG_ENDIAN
-        || encoding == C.ENCODING_PCM_FLOAT;
+        || encoding == C.ENCODING_PCM_FLOAT
+        || encoding == C.ENCODING_PCM_DOUBLE;
   }
 
   /**
@@ -2273,7 +2287,8 @@ public static boolean isEncodingHighResolutionPcm(@C.PcmEncoding int encoding) {
         || encoding == C.ENCODING_PCM_24BIT_BIG_ENDIAN
         || encoding == C.ENCODING_PCM_32BIT
         || encoding == C.ENCODING_PCM_32BIT_BIG_ENDIAN
-        || encoding == C.ENCODING_PCM_FLOAT;
+        || encoding == C.ENCODING_PCM_FLOAT
+        || encoding == C.ENCODING_PCM_DOUBLE;
   }
 
   /**
@@ -2380,9 +2395,11 @@ public static int getApiLevelThatAudioFormatIntroducedAudioEncoding(int encoding
         return 28;
       case C.ENCODING_OPUS:
         return 30;
+      case C.ENCODING_PCM_24BIT:
       case C.ENCODING_PCM_32BIT:
         return 31;
       case C.ENCODING_DTS_UHD_P2:
+      case C.ENCODING_DSD:
         return 34;
       default:
         return Integer.MAX_VALUE;
@@ -2422,6 +2439,8 @@ public static int getByteDepth(@C.PcmEncoding int pcmEncoding) {
       case C.ENCODING_PCM_32BIT_BIG_ENDIAN:
       case C.ENCODING_PCM_FLOAT:
         return 4;
+      case C.ENCODING_PCM_DOUBLE:
+        return 8;
       case C.ENCODING_INVALID:
       case Format.NO_VALUE:
       default:

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java

@@ -108,6 +108,7 @@ public class DefaultRenderersFactory implements RenderersFactory {
   private long allowedVideoJoiningTimeMs;
   private boolean enableDecoderFallback;
   private MediaCodecSelector mediaCodecSelector;
+  private boolean pcmEncodingRestrictionLifted;
   private boolean enableFloatOutput;
   private boolean enableAudioTrackPlaybackParams;
   private boolean enableMediaCodecVideoRendererPrewarming;
@@ -223,15 +224,38 @@ public final DefaultRenderersFactory setMediaCodecSelector(
    *
    * <p>The default value is {@code false}.
    *
+   * @deprecated Use {@link #setPcmEncodingRestrictionLifted} instead to allow any encoding, not
+   *     just 32-bit float.
    * @param enableFloatOutput Whether to enable use of floating point audio output, if available.
    * @return This factory, for convenience.
    */
   @CanIgnoreReturnValue
+  @Deprecated
   public final DefaultRenderersFactory setEnableAudioFloatOutput(boolean enableFloatOutput) {
     this.enableFloatOutput = enableFloatOutput;
     return this;
   }
 
+  /**
+   * Sets whether to enable outputting samples in any platform-supported format (such as 32-bit
+   * float, 32-bit integer, 24-bit integer, 16-bit integer or 8-bit integer) instead of restricting
+   * output to 16-bit integers. Where possible, the input sample format will be used, otherwise
+   * high-resolution formats will be output as 32-bit float. Parts of the default audio processing
+   * chain (for example, speed adjustment) will not be available when output formats other than
+   * 16-bit integer are in use.
+   *
+   * <p>The default value is {@code false}.
+   *
+   * @param pcmEncodingRestrictionLifted Whether to lift any restriction of output sample format.
+   * @return This factory, for convenience.
+   */
+  @CanIgnoreReturnValue
+  public final DefaultRenderersFactory setPcmEncodingRestrictionLifted(
+      boolean pcmEncodingRestrictionLifted) {
+    this.pcmEncodingRestrictionLifted = pcmEncodingRestrictionLifted;
+    return this;
+  }
+
   /**
    * Sets whether to enable setting playback speed using {@link
    * android.media.AudioTrack#setPlaybackParams(PlaybackParams)}, which is supported from API level
@@ -375,7 +399,11 @@ public Renderer[] createRenderers(
         renderersList);
     @Nullable
     AudioSink audioSink =
-        buildAudioSink(context, enableFloatOutput, enableAudioTrackPlaybackParams);
+        buildAudioSink(
+            context,
+            pcmEncodingRestrictionLifted,
+            enableFloatOutput,
+            enableAudioTrackPlaybackParams);
     if (audioSink != null) {
       buildAudioRenderers(
           context,
@@ -814,11 +842,17 @@ protected void buildMiscellaneousRenderers(
    */
   @Nullable
   protected AudioSink buildAudioSink(
-      Context context, boolean enableFloatOutput, boolean enableAudioTrackPlaybackParams) {
-    return new DefaultAudioSink.Builder(context)
-        .setEnableFloatOutput(enableFloatOutput)
-        .setEnableAudioTrackPlaybackParams(enableAudioTrackPlaybackParams)
-        .build();
+      Context context,
+      boolean pcmEncodingRestrictionLifted,
+      boolean enableFloatOutput,
+      boolean enableAudioTrackPlaybackParams) {
+    DefaultAudioSink.Builder builder = new DefaultAudioSink.Builder(context);
+    if (pcmEncodingRestrictionLifted || !enableFloatOutput) {
+      builder.setPcmEncodingRestrictionLifted(pcmEncodingRestrictionLifted);
+    } else {
+      builder.setEnableFloatOutput(true);
+    }
+    return builder.setEnableAudioTrackPlaybackParams(enableAudioTrackPlaybackParams).build();
   }
 
   @Override

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ChannelMappingAudioProcessor.java

@@ -113,6 +113,9 @@ public void queueInput(ByteBuffer inputBuffer) {
           case C.ENCODING_PCM_FLOAT:
             buffer.putFloat(inputBuffer.getFloat(inputIndex));
             break;
+          case C.ENCODING_PCM_DOUBLE:
+            buffer.putDouble(inputBuffer.getDouble(inputIndex));
+            break;
           default:
             throw new IllegalStateException("Unexpected encoding: " + inputAudioFormat.encoding);
         }