diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java index 31ed5f3..a5abf94 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java @@ -51,6 +51,8 @@ final class VoiceStream { private final float[] pcm = new float[MAX_FRAME * VoiceFormat.MAX_CHANNELS]; private OpusDecoder decoder; private int decoderChannels; + /** Length of the last packet, which a lost one most likely had too. */ + private int lastFrameSize = VoiceFormat.FRAME_SIZE; private boolean closed; private volatile boolean talking; @@ -142,7 +144,13 @@ final class VoiceStream { synchronized (decoderLock) { if (closed) return 0; try { - frames = decoderFor(channels).decode(data, pcm); + OpusDecoder d = decoderFor(channels); + if (data == null) { + frames = d.conceal(pcm, lastFrameSize); + } else { + frames = d.decode(data, pcm); + lastFrameSize = frames; + } } catch (RuntimeException e) { return 0; } diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/opus/OpusDecoder.java b/ts3-client/core/src/main/java/com/ts3client/audio/opus/OpusDecoder.java index 970ca58..f5414f2 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/opus/OpusDecoder.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/opus/OpusDecoder.java @@ -3,12 +3,15 @@ package com.ts3client.audio.opus; /** One Opus decoder, used from one thread at a time. */ public interface OpusDecoder extends AutoCloseable { - /** - * Decodes {@code packet} into interleaved samples, or conceals a lost packet when it is - * {@code null}. Returns the samples decoded per channel. - */ + /** Decodes {@code packet} into interleaved samples; returns the samples decoded per channel. */ int decode(byte[] packet, float[] out); + /** + * Makes up {@code frameSize} samples per channel for a packet that never arrived. Opus + * conceals exactly as much as it is asked for, so this should be the lost packet's length. + */ + int conceal(float[] out, int frameSize); + /** Forgets the stream's state, as at the start of a new talk burst. */ void reset(); diff --git a/ts3-client/core/src/test/java/com/ts3client/audio/FakeOpus.java b/ts3-client/core/src/test/java/com/ts3client/audio/FakeOpus.java index 7388f89..c91baf2 100644 --- a/ts3-client/core/src/test/java/com/ts3client/audio/FakeOpus.java +++ b/ts3-client/core/src/test/java/com/ts3client/audio/FakeOpus.java @@ -4,17 +4,20 @@ import com.ts3client.audio.opus.OpusCodec; import com.ts3client.audio.opus.OpusDecoder; import com.ts3client.audio.opus.OpusEncoder; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; /** * Stands in for libopus: a "packet" is one byte, decoded to a 20 ms frame holding that byte - * divided by 100 in every sample. Concealment decodes to -1, so it is easy to spot. + * divided by 100 in every sample. Concealment yields -1, so it is easy to spot. */ final class FakeOpus implements OpusCodec { static final float CONCEALED = -1f; int resets; + final List concealedFrameSizes = new ArrayList<>(); static byte[] packet(int value) { return new byte[]{(byte) value}; @@ -30,11 +33,17 @@ final class FakeOpus implements OpusCodec { return new OpusDecoder() { @Override public int decode(byte[] packet, float[] out) { - float v = packet == null ? CONCEALED : packet[0] / 100f; - Arrays.fill(out, 0, VoiceFormat.FRAME_SIZE * channels, v); + Arrays.fill(out, 0, VoiceFormat.FRAME_SIZE * channels, packet[0] / 100f); return VoiceFormat.FRAME_SIZE; } + @Override + public int conceal(float[] out, int frameSize) { + concealedFrameSizes.add(frameSize); + Arrays.fill(out, 0, frameSize * channels, CONCEALED); + return frameSize; + } + @Override public void reset() { resets++; diff --git a/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java b/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java index 072699e..423f672 100644 --- a/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java +++ b/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java @@ -50,6 +50,8 @@ class VoiceStreamTest { assertEquals(0.01f, pull()); assertEquals(FakeOpus.CONCEALED, pull()); assertEquals(0.03f, pull()); + assertEquals(List.of(VoiceFormat.FRAME_SIZE), opus.concealedFrameSizes, + "a lost packet is concealed at the length of the one before, not the longest Opus frame"); } @Test diff --git a/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/NativeOpusDecoder.java b/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/NativeOpusDecoder.java index 55ba7db..de3dd5a 100644 --- a/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/NativeOpusDecoder.java +++ b/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/NativeOpusDecoder.java @@ -43,29 +43,27 @@ final class NativeOpusDecoder implements OpusDecoder { } /** - * Decodes an Opus packet to interleaved float PCM. - * - * @param packet the encoded packet, or {@code null} to request packet-loss - * concealment (PLC) for a missing frame + * @param packet an Opus packet, at most {@value #MAX_PACKET_BYTES} bytes * @param out output buffer, at least {@code frameSize * channels} long * @return number of samples decoded per channel */ @Override public int decode(byte[] packet, float[] out) { - if (closed) throw new IllegalStateException("decoder closed"); - - MemorySegment data = MemorySegment.NULL; - int len = 0; - if (packet != null) { - if (packet.length > MAX_PACKET_BYTES) { - throw new IllegalArgumentException("packet too large: " + packet.length); - } - MemorySegment.copy(packet, 0, packetBuffer, ValueLayout.JAVA_BYTE, 0, packet.length); - data = packetBuffer; - len = packet.length; + if (packet.length > MAX_PACKET_BYTES) { + throw new IllegalArgumentException("packet too large: " + packet.length); } + MemorySegment.copy(packet, 0, packetBuffer, ValueLayout.JAVA_BYTE, 0, packet.length); + return decode(packetBuffer, packet.length, out, frameSize); + } - int samples = Opus.decodeFloat(handle, data, len, pcmBuffer, frameSize, 0); + @Override + public int conceal(float[] out, int frameSize) { + return decode(MemorySegment.NULL, 0, out, Math.min(frameSize, this.frameSize)); + } + + private int decode(MemorySegment data, int len, float[] out, int maxSamples) { + if (closed) throw new IllegalStateException("decoder closed"); + int samples = Opus.decodeFloat(handle, data, len, pcmBuffer, maxSamples, 0); if (samples < 0) { throw new IllegalStateException("opus_decode_float failed: " + Opus.strerror(samples)); }