From 70aac6f5ce5dccf27760a676b8dd79bb816f12ca Mon Sep 17 00:00:00 2001 From: ericek111 Date: Tue, 6 Oct 2026 20:38:20 +0000 Subject: [PATCH] Add extra receive buffering and decode Opus FEC on loss A new Extra buffering setting (Network page) holds more playback delay than the jitter buffer measures it needs, so late packets still make it in time on a bad connection. A lost frame is now rebuilt from the FEC copy in the next packet when that one is already held, and the Connection Info shows how many were recovered. Held packets' arrival marks now move with the timeline when the delay changes; libspeex leaves them, which made a multi-frame jump count twice. Co-Authored-By: Claude Opus 5.5 --- .../ts3client/android/SessionController.kt | 1 + .../android/ui/NetworkSettingsScreen.kt | 9 +++- .../ts3client/android/ui/SettingsScreen.kt | 2 +- android/audio/src/main/cpp/opus_jni.cpp | 9 ++-- .../ts3client/android/audio/NativeOpus.java | 11 +++-- .../com/ts3client/audio/JitterBuffer.java | 13 +++++- .../ts3client/audio/StreamingVoiceOutput.java | 10 ++++ .../java/com/ts3client/audio/VoiceOutput.java | 6 +++ .../ts3client/audio/VoiceReceiveStats.java | 12 +++-- .../java/com/ts3client/audio/VoiceStream.java | 46 +++++++++++++++++-- .../com/ts3client/audio/opus/OpusDecoder.java | 8 ++++ .../java/com/ts3client/config/Settings.java | 9 ++++ .../ts3client/net/ConnectionStatsFormat.java | 1 + .../ts3client/net/TeamspeakConnection.java | 1 + .../java/com/ts3client/audio/FakeOpus.java | 9 +++- .../com/ts3client/audio/JitterBufferTest.java | 21 +++++++++ .../com/ts3client/audio/VoiceStreamTest.java | 29 ++++++++++-- .../audio/desktop/NativeOpusDecoder.java | 17 +++++-- .../main/java/com/ts3client/ui/MainFrame.java | 3 +- .../java/com/ts3client/ui/NetworkPanel.java | 32 ++++++++----- 20 files changed, 210 insertions(+), 39 deletions(-) diff --git a/android/app/src/main/java/com/ts3client/android/SessionController.kt b/android/app/src/main/java/com/ts3client/android/SessionController.kt index a15dedb..0845ef4 100644 --- a/android/app/src/main/java/com/ts3client/android/SessionController.kt +++ b/android/app/src/main/java/com/ts3client/android/SessionController.kt @@ -420,6 +420,7 @@ class SessionController(private val context: Context) { for (server in _servers.value) { server.playback?.setMasterVolume(settings.outputVolume) server.playback?.setOutputDevice(settings.outputDevice) + server.playback?.setReceiveBuffer(settings.receiveBuffer) server.microphone?.apply(settings) server.refresh() server.scheduleTree(structure = false) diff --git a/android/app/src/main/java/com/ts3client/android/ui/NetworkSettingsScreen.kt b/android/app/src/main/java/com/ts3client/android/ui/NetworkSettingsScreen.kt index 98ba792..4ac1038 100644 --- a/android/app/src/main/java/com/ts3client/android/ui/NetworkSettingsScreen.kt +++ b/android/app/src/main/java/com/ts3client/android/ui/NetworkSettingsScreen.kt @@ -18,7 +18,7 @@ import kotlin.math.roundToInt private const val MIN_BITRATE_KBITS = 8f private const val MAX_BITRATE_KBITS = 160f -/** What voice costs and survives on the wire: bitrate, error correction and duplicate packets. */ +/** What voice costs and survives on the wire: bitrate, error correction, duplicate packets and receive buffering. */ @Composable fun NetworkSettingsScreen(vm: SessionController, onBack: () -> Unit) { val settings = vm.settings @@ -30,6 +30,7 @@ fun NetworkSettingsScreen(vm: SessionController, onBack: () -> Unit) { var fec by remember { mutableStateOf(settings.fec) } var redundancy by remember { mutableStateOf(settings.redundancyDelay) } var copies by remember { mutableStateOf(settings.redundancyCopies) } + var receiveBuffer by remember { mutableStateOf(settings.receiveBuffer) } SubScreen("Network", onBack) { padding -> Column(Modifier.padding(padding).verticalScroll(rememberScrollState()).padding(bottom = 8.dp)) { @@ -57,6 +58,12 @@ fun NetworkSettingsScreen(vm: SessionController, onBack: () -> Unit) { change { redundancyCopies = it } } } + + Section("Receive") + Picker("Extra buffering", Settings.RECEIVE_BUFFERS.map { it to if (it == 0) "Off" else "$it ms" }, receiveBuffer) { + receiveBuffer = it + change { this.receiveBuffer = it } + } } } } diff --git a/android/app/src/main/java/com/ts3client/android/ui/SettingsScreen.kt b/android/app/src/main/java/com/ts3client/android/ui/SettingsScreen.kt index 30b5b3a..44f3b2c 100644 --- a/android/app/src/main/java/com/ts3client/android/ui/SettingsScreen.kt +++ b/android/app/src/main/java/com/ts3client/android/ui/SettingsScreen.kt @@ -45,7 +45,7 @@ fun SettingsScreen(vm: SessionController, onContacts: () -> Unit, onBack: () -> LinkRow("Sound Settings", "Devices, transmission, voice activation, processing, volume", "PLAYBACK") { page = SettingsPage.SOUND } - LinkRow("Network", "Bitrate, error correction, redundancy", "PING_4") { + LinkRow("Network", "Bitrate, error correction, redundancy, buffering", "PING_4") { page = SettingsPage.NETWORK } LinkRow("Design", "Icon pack, country flags, description pictures", "GUISETUP") { diff --git a/android/audio/src/main/cpp/opus_jni.cpp b/android/audio/src/main/cpp/opus_jni.cpp index 3008540..248f36a 100644 --- a/android/audio/src/main/cpp/opus_jni.cpp +++ b/android/audio/src/main/cpp/opus_jni.cpp @@ -74,10 +74,13 @@ Java_com_ts3client_android_audio_NativeOpus_decoderCreate(JNIEnv *env, jclass, j return reinterpret_cast(created); } -/** Decodes {@code packet}, or conceals {@code frameSize} samples when it is null. */ +/** + * Decodes {@code packet}, or conceals {@code frameSize} samples when it is null; with {@code fec}, + * rebuilds the packet before {@code packet} from its error correction copy. + */ JNIEXPORT jint JNICALL Java_com_ts3client_android_audio_NativeOpus_decode(JNIEnv *env, jclass, jlong handle, jbyteArray packet, - jfloatArray out, jint frameSize) { + jfloatArray out, jint frameSize, jboolean fec) { std::vector data; if (packet != nullptr) { data.resize(static_cast(env->GetArrayLength(packet))); @@ -85,7 +88,7 @@ Java_com_ts3client_android_audio_NativeOpus_decode(JNIEnv *env, jclass, jlong ha } jfloat *samples = env->GetFloatArrayElements(out, nullptr); int decoded = opus_decode_float(decoder(handle), packet != nullptr ? data.data() : nullptr, - static_cast(data.size()), samples, frameSize, 0); + static_cast(data.size()), samples, frameSize, fec ? 1 : 0); env->ReleaseFloatArrayElements(out, samples, decoded < 0 ? JNI_ABORT : 0); if (decoded < 0) throwOpus(env, "opus_decode_float", decoded); return decoded; diff --git a/android/audio/src/main/java/com/ts3client/android/audio/NativeOpus.java b/android/audio/src/main/java/com/ts3client/android/audio/NativeOpus.java index 0673198..5abbb56 100644 --- a/android/audio/src/main/java/com/ts3client/android/audio/NativeOpus.java +++ b/android/audio/src/main/java/com/ts3client/android/audio/NativeOpus.java @@ -68,12 +68,17 @@ public final class NativeOpus implements OpusCodec { @Override public int decode(byte[] packet, float[] out) { - return NativeOpus.decode(live(), packet, out, maxFrameSize); + return NativeOpus.decode(live(), packet, out, maxFrameSize, false); } @Override public int conceal(float[] out, int frameSize) { - return NativeOpus.decode(live(), null, out, Math.min(frameSize, maxFrameSize)); + return NativeOpus.decode(live(), null, out, Math.min(frameSize, maxFrameSize), false); + } + + @Override + public int recover(byte[] next, float[] out, int frameSize) { + return NativeOpus.decode(live(), next, out, Math.min(frameSize, maxFrameSize), true); } @Override @@ -104,7 +109,7 @@ public final class NativeOpus implements OpusCodec { private static native long decoderCreate(int sampleRate, int channels); - private static native int decode(long handle, byte[] packet, float[] out, int frameSize); + private static native int decode(long handle, byte[] packet, float[] out, int frameSize, boolean fec); private static native void decoderReset(long handle); diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/JitterBuffer.java b/ts3-client/core/src/main/java/com/ts3client/audio/JitterBuffer.java index 2b24758..05afcd7 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/JitterBuffer.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/JitterBuffer.java @@ -184,8 +184,13 @@ public final class JitterBuffer { this.subWindowSize = this.windowSize / SUB_WINDOWS; } - /** Sets the extra delay held on top of the measured optimum. */ + /** + * Sets the extra delay held on top of the measured optimum. The timings already recorded + * are re-based onto it, so the delay moves at the next {@link #tick} rather than once the + * estimate has turned over. + */ public void setMargin(int margin) { + shiftTimings(bufferMargin - margin); this.bufferMargin = margin; } @@ -478,6 +483,12 @@ public final class JitterBuffer { shiftTimings(-optimal); pointerTimestamp += optimal; if (optimal < 0) interpolationRequested = -optimal; + // Held packets measure their slack against when they arrived, so move that with the + // timeline too. libspeex does not, which makes a delay jump of several frames count + // twice: every packet already waiting reports the old shortfall again on playout. + for (int i = 0; i < MAX_PACKETS; i++) { + if (data[i] != null && arrival[i] != 0) arrival[i] += optimal; + } } /** diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/StreamingVoiceOutput.java b/ts3-client/core/src/main/java/com/ts3client/audio/StreamingVoiceOutput.java index 53deb65..5a25131 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/StreamingVoiceOutput.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/StreamingVoiceOutput.java @@ -47,6 +47,8 @@ public final class StreamingVoiceOutput implements VoiceOutput { private volatile String outputDevice; private volatile double masterVolume = 1.0; + /** Extra playback delay, in jitter buffer ticks. */ + private volatile int extraDelayTicks; private volatile boolean deafened = false; /** Notified (clientId, talking) on an audio thread when a speaker starts/stops. */ @@ -149,6 +151,13 @@ public final class StreamingVoiceOutput implements VoiceOutput { this.outputDevice = device == null ? "" : device; } + @Override + public void setReceiveBuffer(int ms) { + int frames = Math.max(0, ms) * (VoiceFormat.SAMPLE_RATE / 1000) / VoiceFormat.FRAME_SIZE; + extraDelayTicks = frames * JitterBuffer.TEAMSPEAK_TICKS_PER_FRAME; + for (VoiceStream s : streams.values()) s.setExtraDelay(extraDelayTicks); + } + /** Entry point wired into {@code client.setVoiceHandler(...)}. */ @Override public void handleVoice(PacketBody0Voice voice) { @@ -168,6 +177,7 @@ public final class StreamingVoiceOutput implements VoiceOutput { VoiceStream stream = streams.computeIfAbsent(clientId, id -> { VoiceStream created = new VoiceStream(id, opus, this::fireTalk); created.setTap(voiceTap); + created.setExtraDelay(extraDelayTicks); return created; }); if (stream.offer(packetId, codec, data)) playout.started(stream); diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceOutput.java b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceOutput.java index e80be8c..d6bfd35 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceOutput.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceOutput.java @@ -34,6 +34,12 @@ public interface VoiceOutput { void setOutputDevice(String device); + /** + * Delays every speaker by {@code ms} more than the network calls for, trading latency for + * fewer dropouts on a bad connection. + */ + void setReceiveBuffer(int ms); + void removeClient(int clientId); void shutdown(); diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceReceiveStats.java b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceReceiveStats.java index 70a9f77..847d7f6 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceReceiveStats.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceReceiveStats.java @@ -6,23 +6,25 @@ package com.ts3client.audio; * * @param received voice packets that arrived, copies included * @param played frames played from a packet - * @param missed frames concealed because their packet was not there when due + * @param missed frames whose packet was not there when due + * @param recovered of those, frames rebuilt from the error correction copy in the next packet + * rather than concealed * @param outOfOrder packets that arrived after a later one * @param late packets that arrived after their frame was concealed, so were dropped; * each one is also among {@code missed} * @param duplicates copies of a packet already received, discarded */ -public record VoiceReceiveStats(long received, long played, long missed, long outOfOrder, long late, +public record VoiceReceiveStats(long received, long played, long missed, long recovered, long outOfOrder, long late, long duplicates) { - public static final VoiceReceiveStats NONE = new VoiceReceiveStats(0, 0, 0, 0, 0, 0); + public static final VoiceReceiveStats NONE = new VoiceReceiveStats(0, 0, 0, 0, 0, 0, 0); public VoiceReceiveStats plus(VoiceReceiveStats o) { return new VoiceReceiveStats(received + o.received, played + o.played, missed + o.missed, - outOfOrder + o.outOfOrder, late + o.late, duplicates + o.duplicates); + recovered + o.recovered, outOfOrder + o.outOfOrder, late + o.late, duplicates + o.duplicates); } - /** Frames that were due, whether played or concealed. */ + /** Frames that were due, whether played or missed. */ public long due() { return played + missed; } diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java index be1c83b..d0e9b80 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java @@ -20,6 +20,11 @@ import java.util.function.BiConsumer; * position is remembered, the burst ends once playback reaches it, and its slot is played as * silence rather than concealed, since nothing was lost there. * + *

A frame that did go missing is rebuilt from the error correction copy in the packet after + * it, when that one is already held; only otherwise is it concealed. An + * {@linkplain #setExtraDelay extra delay} makes that, and late packets, more likely to be in + * time. + * *

The stream keeps no clock of its own: whoever pulls sets the pace, one call per * 20 ms frame. {@link #offer} and {@link #stop} may be called from any thread; pulls * must come from one thread at a time. @@ -46,7 +51,7 @@ final class VoiceStream { private static final int SEEN_WINDOW = 1024; /** What a pull plays, decided under the lock and produced outside it. */ - private enum Source { PACKET, CONCEAL, SILENCE, SURPLUS } + private enum Source { PACKET, RECOVER, CONCEAL, SILENCE, SURPLUS } final int clientId; private final OpusCodec opus; @@ -67,6 +72,10 @@ final class VoiceStream { private int surplusFrames; /** Frames of concealment still owed to a delay increase. */ private int concealOwed; + /** Ticks of delay held on top of what the jitter buffer measures it needs. */ + private int extraTicks; + /** Frames of silence still to play before a fresh timeline starts, building up {@link #extraTicks}. */ + private int leadInFrames; /** Set when the timeline was dropped; the decoder follows on the next pull. */ private boolean resetDecoder; /** Recently received sequence numbers, each at {@code seq % SEEN_WINDOW}. */ @@ -75,6 +84,7 @@ final class VoiceStream { private long received; private long played; private long missed; + private long recovered; private long outOfOrder; private long late; private long duplicates; @@ -107,6 +117,17 @@ final class VoiceStream { this.tap = tap; } + /** + * Holds {@code ticks} more delay than the network calls for, for a better chance that + * late packets and those carrying a lost one's error correction copy arrive in time. + */ + void setExtraDelay(int ticks) { + synchronized (lock) { + extraTicks = Math.max(0, ticks); + jitter.setMargin(FRAME_TICKS + extraTicks); + } + } + private static int channelsFor(CodecType codec) { return codec == CodecType.OPUS_MUSIC ? 2 : 1; } @@ -154,6 +175,8 @@ final class VoiceStream { playing = true; started = true; emptyPulls = 0; + // The buffer would otherwise reach the extra delay by concealing frames mid-speech. + if (fresh) leadInFrames = extraTicks / FRAME_TICKS; } } markTalking(true); @@ -180,7 +203,10 @@ final class VoiceStream { resetDecoder = false; packetChannels = channels; - if (surplusFrames > 0 || concealOwed > 0) { + if (leadInFrames > 0) { + leadInFrames--; + source = Source.SILENCE; + } else if (surplusFrames > 0 || concealOwed > 0) { // Still playing out what the last get() handed over. if (surplusFrames > 0) { source = Source.SURPLUS; @@ -209,8 +235,14 @@ final class VoiceStream { if (isStopMark(jitter.timestamp())) { source = Source.SILENCE; } else { - source = Source.CONCEAL; missed++; + packet = jitter.nextPayload(); + if (packet != null) { + source = Source.RECOVER; + recovered++; + } else { + source = Source.CONCEAL; + } } } default -> { @@ -258,6 +290,11 @@ final class VoiceStream { play(pcm, 0, frames, decoderChannels, out, outChannels, gain); return frames; } + case RECOVER -> { + int frames = decoderFor(decoderChannels == 0 ? packetChannels : decoderChannels).recover(packet, pcm, FRAME); + play(pcm, 0, frames, decoderChannels, out, outChannels, gain); + return frames; + } default -> { int frames = decoderFor(packetChannels).decode(packet, pcm); int played = Math.min(frames, FRAME); @@ -372,7 +409,7 @@ final class VoiceStream { /** How this speaker's voice has been coming in since the stream was created. */ VoiceReceiveStats stats() { synchronized (lock) { - return new VoiceReceiveStats(received, played, missed, outOfOrder, late, duplicates); + return new VoiceReceiveStats(received, played, missed, recovered, outOfOrder, late, duplicates); } } @@ -415,6 +452,7 @@ final class VoiceStream { playing = false; surplusFrames = 0; concealOwed = 0; + leadInFrames = 0; emptyPulls = 0; } diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/opus/OpusDecoder.java b/ts3-client/core/src/main/java/com/ts3client/audio/opus/OpusDecoder.java index f5414f2..b2c4e77 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/opus/OpusDecoder.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/opus/OpusDecoder.java @@ -12,6 +12,14 @@ public interface OpusDecoder extends AutoCloseable { */ int conceal(float[] out, int frameSize); + /** + * Rebuilds {@code frameSize} samples per channel for a lost packet from the forward error + * correction copy carried in {@code next}, the packet that followed it. Falls back to + * concealment by itself when {@code next} carries none. {@code next} is then decoded as + * usual. + */ + int recover(byte[] next, float[] out, int frameSize); + /** Forgets the stream's state, as at the start of a new talk burst. */ void reset(); diff --git a/ts3-client/core/src/main/java/com/ts3client/config/Settings.java b/ts3-client/core/src/main/java/com/ts3client/config/Settings.java index 5080aca..1ee48db 100644 --- a/ts3-client/core/src/main/java/com/ts3client/config/Settings.java +++ b/ts3-client/core/src/main/java/com/ts3client/config/Settings.java @@ -133,6 +133,13 @@ public final class Settings { /** How many times every voice packet is sent again. */ public int redundancyCopies = 1; public static final int MAX_REDUNDANCY_COPIES = 4; + /** + * Milliseconds of playback delay held on top of what the jitter buffer measures it needs, + * so late packets and error correction copies still make it on a bad connection. + */ + public int receiveBuffer = 0; + /** The {@link #receiveBuffer} choices the settings offer; whole 20 ms voice frames. */ + public static final int[] RECEIVE_BUFFERS = {0, 20, 40, 60, 80, 120, 200, 300, 500}; /** Master playback gain, 0..1 (may exceed 1 for boost up to 2). */ public double outputVolume = 1.0; /** @@ -291,6 +298,7 @@ public final class Settings { music = parseB(props.getProperty("music"), music); redundancyDelay = parseI(props.getProperty("redundancyDelay"), redundancyDelay); redundancyCopies = parseI(props.getProperty("redundancyCopies"), redundancyCopies); + receiveBuffer = parseI(props.getProperty("receiveBuffer"), receiveBuffer); outputVolume = parseD(props.getProperty("outputVolume"), outputVolume); masterVolume = parseD(props.getProperty("masterVolume"), masterVolume); inputVolume = parseD(props.getProperty("inputVolume"), inputVolume); @@ -351,6 +359,7 @@ public final class Settings { props.setProperty("music", Boolean.toString(music)); props.setProperty("redundancyDelay", Integer.toString(redundancyDelay)); props.setProperty("redundancyCopies", Integer.toString(redundancyCopies)); + props.setProperty("receiveBuffer", Integer.toString(receiveBuffer)); props.setProperty("outputVolume", Double.toString(outputVolume)); props.setProperty("masterVolume", Double.toString(masterVolume)); props.setProperty("inputVolume", Double.toString(inputVolume)); diff --git a/ts3-client/core/src/main/java/com/ts3client/net/ConnectionStatsFormat.java b/ts3-client/core/src/main/java/com/ts3client/net/ConnectionStatsFormat.java index 817f730..791216f 100644 --- a/ts3-client/core/src/main/java/com/ts3client/net/ConnectionStatsFormat.java +++ b/ts3-client/core/src/main/java/com/ts3client/net/ConnectionStatsFormat.java @@ -121,6 +121,7 @@ public final class ConnectionStatsFormat { new Row("Received", known ? count(s.received()) : UNKNOWN), new Row("Played", known ? countOf(s.played(), s.due()) : UNKNOWN), new Row("Missed", known ? countOf(s.missed(), s.due()) : UNKNOWN), + new Row("Recovered (FEC)", known ? countOf(s.recovered(), s.missed()) : UNKNOWN), new Row("Out of order", known ? countOf(s.outOfOrder(), s.received()) : UNKNOWN), new Row("Dropped (too late)", known ? countOf(s.late(), s.received()) : UNKNOWN), new Row("Discarded (copies)", known ? countOf(s.duplicates(), s.received()) : UNKNOWN)); diff --git a/ts3-client/core/src/main/java/com/ts3client/net/TeamspeakConnection.java b/ts3-client/core/src/main/java/com/ts3client/net/TeamspeakConnection.java index 8598b06..250b648 100644 --- a/ts3-client/core/src/main/java/com/ts3client/net/TeamspeakConnection.java +++ b/ts3-client/core/src/main/java/com/ts3client/net/TeamspeakConnection.java @@ -285,6 +285,7 @@ public final class TeamspeakConnection implements TS3Listener { model.clear(); playback = audio.createOutput(settings); playback.setMasterVolume(settings.effectiveOutputVolume()); + playback.setReceiveBuffer(settings.receiveBuffer); playback.setTalkListener((clientId, talking) -> { ClientEntry c = model.getClient(clientId); if (c != null) c.talking = talking; diff --git a/ts3-client/core/src/test/java/com/ts3client/audio/FakeOpus.java b/ts3-client/core/src/test/java/com/ts3client/audio/FakeOpus.java index ab0139a..e8d8769 100644 --- a/ts3-client/core/src/test/java/com/ts3client/audio/FakeOpus.java +++ b/ts3-client/core/src/test/java/com/ts3client/audio/FakeOpus.java @@ -10,11 +10,12 @@ import java.util.List; /** * Stands in for libopus: a "packet" of n bytes decodes to n 20 ms frames holding the first byte - * divided by 100 in every sample. Concealment yields -1, so it is easy to spot. + * divided by 100 in every sample. Concealment yields -1 and recovery -2, so both are easy to spot. */ final class FakeOpus implements OpusCodec { static final float CONCEALED = -1f; + static final float RECOVERED = -2f; int resets; final List concealedFrameSizes = new ArrayList<>(); @@ -45,6 +46,12 @@ final class FakeOpus implements OpusCodec { return frameSize; } + @Override + public int recover(byte[] next, float[] out, int frameSize) { + Arrays.fill(out, 0, frameSize * channels, RECOVERED); + return frameSize; + } + @Override public void reset() { resets++; diff --git a/ts3-client/core/src/test/java/com/ts3client/audio/JitterBufferTest.java b/ts3-client/core/src/test/java/com/ts3client/audio/JitterBufferTest.java index 8d77387..9fdff01 100644 --- a/ts3-client/core/src/test/java/com/ts3client/audio/JitterBufferTest.java +++ b/ts3-client/core/src/test/java/com/ts3client/audio/JitterBufferTest.java @@ -51,6 +51,27 @@ class JitterBufferTest { assertTrue(real.size() >= 15, "most frames should have played: " + played); } + @Test + void aLargerMarginBuffersThatMuchMore() { + JitterBuffer b = buffer(); + int n = 0; + int lastPlayed = -1; + for (; n < 60; n++) { + put(b, n); + int tag = pull(b); + if (tag >= 0) lastPlayed = tag; + } + int before = n - 1 - lastPlayed; + + b.setMargin(b.margin() + 5 * SPAN); + for (; n < 120; n++) { + put(b, n); + int tag = pull(b); + if (tag >= 0) lastPlayed = tag; + } + assertEquals(before + 5, n - 1 - lastPlayed); + } + @Test void reordersPacketsThatArriveOutOfOrder() { JitterBuffer b = buffer(); diff --git a/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java b/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java index 97d1eff..149e6a4 100644 --- a/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java +++ b/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java @@ -59,15 +59,38 @@ class VoiceStreamTest { } @Test - void concealsALostPacket() { + void rebuildsALostPacketFromTheOneAfterIt() { offer(0, 1); offer(2, 3); assertEquals(0.01f, pull()); - assertEquals(FakeOpus.CONCEALED, pull()); + assertEquals(FakeOpus.RECOVERED, pull()); assertEquals(0.03f, pull()); + assertEquals(List.of(), opus.concealedFrameSizes); + } + + @Test + void concealsALostPacketWhenTheNextIsMissingToo() { + offer(0, 1); + offer(3, 4); + assertEquals(0.01f, pull()); + assertEquals(FakeOpus.CONCEALED, pull()); + assertEquals(FakeOpus.RECOVERED, pull()); + assertEquals(0.04f, pull()); assertEquals(List.of(VoiceFormat.FRAME_SIZE), opus.concealedFrameSizes); } + @Test + void anExtraDelayLeadsANewSpeakerInWithSilence() { + stream.setExtraDelay(2 * JitterBuffer.TEAMSPEAK_TICKS_PER_FRAME); + offer(0, 1); + assertEquals(0f, pull()); + offer(1, 2); + assertEquals(0f, pull()); + assertEquals(0.01f, pull()); + assertEquals(0.02f, pull()); + assertEquals(List.of(), opus.concealedFrameSizes); + } + @Test void dropsPacketsThatArriveTooLate() { offer(0, 1); @@ -100,7 +123,7 @@ class VoiceStreamTest { pull(); // 3 never arrives: concealed offer(3, 4); // too late, dropped offer(3, 4); // a copy of that - assertEquals(new VoiceReceiveStats(6, 3, 1, 1, 1, 2), stream.stats()); + assertEquals(new VoiceReceiveStats(6, 3, 1, 0, 1, 1, 2), stream.stats()); } @Test diff --git a/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/NativeOpusDecoder.java b/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/NativeOpusDecoder.java index 1954bc8..c1c5615 100644 --- a/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/NativeOpusDecoder.java +++ b/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/NativeOpusDecoder.java @@ -54,17 +54,26 @@ final class NativeOpusDecoder implements OpusDecoder { throw new IllegalArgumentException("packet too large: " + packet.length); } MemorySegment.copy(packet, 0, packetBuffer, ValueLayout.JAVA_BYTE, 0, packet.length); - return decode(packetBuffer, packet.length, out, frameSize); + return decode(packetBuffer, packet.length, out, frameSize, 0); } @Override public int conceal(float[] out, int frameSize) { - return decode(MemorySegment.NULL, 0, out, Math.min(frameSize, this.frameSize)); + return decode(MemorySegment.NULL, 0, out, Math.min(frameSize, this.frameSize), 0); } - private int decode(MemorySegment data, int len, float[] out, int maxSamples) { + @Override + public int recover(byte[] next, float[] out, int frameSize) { + if (next.length > MAX_PACKET_BYTES) { + throw new IllegalArgumentException("packet too large: " + next.length); + } + MemorySegment.copy(next, 0, packetBuffer, ValueLayout.JAVA_BYTE, 0, next.length); + return decode(packetBuffer, next.length, out, Math.min(frameSize, this.frameSize), 1); + } + + private int decode(MemorySegment data, int len, float[] out, int maxSamples, int fec) { if (closed) throw new IllegalStateException("decoder closed"); - int samples = Opus.decodeFloat(handle, data, len, pcmBuffer, maxSamples, 0); + int samples = Opus.decodeFloat(handle, data, len, pcmBuffer, maxSamples, fec); if (samples < 0) { throw new IllegalStateException("opus_decode_float failed: " + Opus.strerror(samples)); } diff --git a/ts3-client/swing/src/main/java/com/ts3client/ui/MainFrame.java b/ts3-client/swing/src/main/java/com/ts3client/ui/MainFrame.java index 9b2bf94..bef1f3c 100644 --- a/ts3-client/swing/src/main/java/com/ts3client/ui/MainFrame.java +++ b/ts3-client/swing/src/main/java/com/ts3client/ui/MainFrame.java @@ -964,13 +964,14 @@ public final class MainFrame extends JFrame implements ServerTabPane.Listener { dlg.setVisible(true); } - /** Master volume / output device are global, so push them to every open connection. */ + /** Master volume, output device and receive buffer are global, so push them to every open connection. */ private void applyOutputSettingsToAllTabs() { soundPlayer.setOutputDevice(settings.outputDevice); for (ServerTab tab : tabs) { if (tab.connection().getPlayback() == null) continue; tab.connection().getPlayback().setMasterVolume(settings.effectiveOutputVolume()); tab.connection().getPlayback().setOutputDevice(settings.outputDevice); + tab.connection().getPlayback().setReceiveBuffer(settings.receiveBuffer); } } diff --git a/ts3-client/swing/src/main/java/com/ts3client/ui/NetworkPanel.java b/ts3-client/swing/src/main/java/com/ts3client/ui/NetworkPanel.java index 3dc66d7..c718859 100644 --- a/ts3-client/swing/src/main/java/com/ts3client/ui/NetworkPanel.java +++ b/ts3-client/swing/src/main/java/com/ts3client/ui/NetworkPanel.java @@ -11,8 +11,8 @@ import java.awt.GridBagConstraints; /** * Options page for what voice costs and survives on the wire: the encoder's bitrate and - * error correction, and duplicate voice packets. Changes preview live on the connected - * microphone. + * error correction, duplicate voice packets, and how long incoming voice is buffered. + * Transmit changes preview live on the connected microphone. */ final class NetworkPanel extends FormPanel { @@ -26,6 +26,7 @@ final class NetworkPanel extends FormPanel { private final JCheckBox fecCheck; private final JComboBox redundancyCombo; private final JComboBox copiesCombo; + private final JComboBox receiveBufferCombo; /** @param pushEncoder puts the encoder settings on screen, across pages, into effect */ NetworkPanel(Settings settings, VoiceInput liveMic, Runnable pushEncoder) { @@ -45,13 +46,8 @@ final class NetworkPanel extends FormPanel { vbrCheck.addActionListener(e -> pushEncoder.run()); fecCheck.addActionListener(e -> pushEncoder.run()); - String[] delays = new String[Settings.REDUNDANCY_DELAYS.length]; - for (int i = 0; i < delays.length; i++) { - int ms = Settings.REDUNDANCY_DELAYS[i]; - delays[i] = ms == 0 ? "Off" : ms + " ms"; - } - redundancyCombo = new JComboBox<>(delays); - redundancyCombo.setSelectedIndex(nearestDelay(settings.redundancyDelay)); + redundancyCombo = new JComboBox<>(delayLabels(Settings.REDUNDANCY_DELAYS)); + redundancyCombo.setSelectedIndex(nearest(Settings.REDUNDANCY_DELAYS, settings.redundancyDelay)); Integer[] copies = new Integer[Settings.MAX_REDUNDANCY_COPIES]; for (int i = 0; i < copies.length; i++) copies[i] = i + 1; copiesCombo = new JComboBox<>(copies); @@ -63,6 +59,9 @@ final class NetworkPanel extends FormPanel { }); copiesCombo.addActionListener(e -> pushRedundancy()); + receiveBufferCombo = new JComboBox<>(delayLabels(Settings.RECEIVE_BUFFERS)); + receiveBufferCombo.setSelectedIndex(nearest(Settings.RECEIVE_BUFFERS, settings.receiveBuffer)); + int row = 0; addSection(this, c, row++, "Encoder"); addRow(this, c, row++, new JLabel("Bitrate:"), sliderWithLabel(bitrateSlider, bitrateLabel, 70)); @@ -71,6 +70,8 @@ final class NetworkPanel extends FormPanel { addSection(this, c, row++, "Redundancy"); addRow(this, c, row++, new JLabel("Delay:"), redundancyCombo); addRow(this, c, row++, new JLabel("Copies:"), copiesCombo); + addSection(this, c, row++, "Receive"); + addRow(this, c, row++, new JLabel("Extra buffering:"), receiveBufferCombo); addFiller(this, c, row); } @@ -82,10 +83,16 @@ final class NetworkPanel extends FormPanel { return (Integer) copiesCombo.getSelectedItem(); } - private static int nearestDelay(int ms) { + private static String[] delayLabels(int[] choices) { + String[] labels = new String[choices.length]; + for (int i = 0; i < labels.length; i++) labels[i] = choices[i] == 0 ? "Off" : choices[i] + " ms"; + return labels; + } + + private static int nearest(int[] choices, int ms) { int best = 0; - for (int i = 1; i < Settings.REDUNDANCY_DELAYS.length; i++) { - if (Math.abs(Settings.REDUNDANCY_DELAYS[i] - ms) < Math.abs(Settings.REDUNDANCY_DELAYS[best] - ms)) best = i; + for (int i = 1; i < choices.length; i++) { + if (Math.abs(choices[i] - ms) < Math.abs(choices[best] - ms)) best = i; } return best; } @@ -101,5 +108,6 @@ final class NetworkPanel extends FormPanel { target.fec = fecCheck.isSelected(); target.redundancyDelay = selectedDelay(); target.redundancyCopies = selectedCopies(); + target.receiveBuffer = Settings.RECEIVE_BUFFERS[receiveBufferCombo.getSelectedIndex()]; } }