diff --git a/android/app/src/main/java/com/ts3client/android/SessionController.kt b/android/app/src/main/java/com/ts3client/android/SessionController.kt index a15dedb..0845ef4 100644 --- a/android/app/src/main/java/com/ts3client/android/SessionController.kt +++ b/android/app/src/main/java/com/ts3client/android/SessionController.kt @@ -420,6 +420,7 @@ class SessionController(private val context: Context) { for (server in _servers.value) { server.playback?.setMasterVolume(settings.outputVolume) server.playback?.setOutputDevice(settings.outputDevice) + server.playback?.setReceiveBuffer(settings.receiveBuffer) server.microphone?.apply(settings) server.refresh() server.scheduleTree(structure = false) diff --git a/android/app/src/main/java/com/ts3client/android/ui/NetworkSettingsScreen.kt b/android/app/src/main/java/com/ts3client/android/ui/NetworkSettingsScreen.kt index 98ba792..4ac1038 100644 --- a/android/app/src/main/java/com/ts3client/android/ui/NetworkSettingsScreen.kt +++ b/android/app/src/main/java/com/ts3client/android/ui/NetworkSettingsScreen.kt @@ -18,7 +18,7 @@ import kotlin.math.roundToInt private const val MIN_BITRATE_KBITS = 8f private const val MAX_BITRATE_KBITS = 160f -/** What voice costs and survives on the wire: bitrate, error correction and duplicate packets. */ +/** What voice costs and survives on the wire: bitrate, error correction, duplicate packets and receive buffering. */ @Composable fun NetworkSettingsScreen(vm: SessionController, onBack: () -> Unit) { val settings = vm.settings @@ -30,6 +30,7 @@ fun NetworkSettingsScreen(vm: SessionController, onBack: () -> Unit) { var fec by remember { mutableStateOf(settings.fec) } var redundancy by remember { mutableStateOf(settings.redundancyDelay) } var copies by remember { mutableStateOf(settings.redundancyCopies) } + var receiveBuffer by remember { mutableStateOf(settings.receiveBuffer) } SubScreen("Network", onBack) { padding -> Column(Modifier.padding(padding).verticalScroll(rememberScrollState()).padding(bottom = 8.dp)) { @@ -57,6 +58,12 @@ fun NetworkSettingsScreen(vm: SessionController, onBack: () -> Unit) { change { redundancyCopies = it } } } + + Section("Receive") + Picker("Extra buffering", Settings.RECEIVE_BUFFERS.map { it to if (it == 0) "Off" else "$it ms" }, receiveBuffer) { + receiveBuffer = it + change { this.receiveBuffer = it } + } } } } diff --git a/android/app/src/main/java/com/ts3client/android/ui/SettingsScreen.kt b/android/app/src/main/java/com/ts3client/android/ui/SettingsScreen.kt index 30b5b3a..44f3b2c 100644 --- a/android/app/src/main/java/com/ts3client/android/ui/SettingsScreen.kt +++ b/android/app/src/main/java/com/ts3client/android/ui/SettingsScreen.kt @@ -45,7 +45,7 @@ fun SettingsScreen(vm: SessionController, onContacts: () -> Unit, onBack: () -> LinkRow("Sound Settings", "Devices, transmission, voice activation, processing, volume", "PLAYBACK") { page = SettingsPage.SOUND } - LinkRow("Network", "Bitrate, error correction, redundancy", "PING_4") { + LinkRow("Network", "Bitrate, error correction, redundancy, buffering", "PING_4") { page = SettingsPage.NETWORK } LinkRow("Design", "Icon pack, country flags, description pictures", "GUISETUP") { diff --git a/android/audio/src/main/cpp/opus_jni.cpp b/android/audio/src/main/cpp/opus_jni.cpp index 3008540..248f36a 100644 --- a/android/audio/src/main/cpp/opus_jni.cpp +++ b/android/audio/src/main/cpp/opus_jni.cpp @@ -74,10 +74,13 @@ Java_com_ts3client_android_audio_NativeOpus_decoderCreate(JNIEnv *env, jclass, j return reinterpret_cast(created); } -/** Decodes {@code packet}, or conceals {@code frameSize} samples when it is null. */ +/** + * Decodes {@code packet}, or conceals {@code frameSize} samples when it is null; with {@code fec}, + * rebuilds the packet before {@code packet} from its error correction copy. + */ JNIEXPORT jint JNICALL Java_com_ts3client_android_audio_NativeOpus_decode(JNIEnv *env, jclass, jlong handle, jbyteArray packet, - jfloatArray out, jint frameSize) { + jfloatArray out, jint frameSize, jboolean fec) { std::vector data; if (packet != nullptr) { data.resize(static_cast(env->GetArrayLength(packet))); @@ -85,7 +88,7 @@ Java_com_ts3client_android_audio_NativeOpus_decode(JNIEnv *env, jclass, jlong ha } jfloat *samples = env->GetFloatArrayElements(out, nullptr); int decoded = opus_decode_float(decoder(handle), packet != nullptr ? data.data() : nullptr, - static_cast(data.size()), samples, frameSize, 0); + static_cast(data.size()), samples, frameSize, fec ? 1 : 0); env->ReleaseFloatArrayElements(out, samples, decoded < 0 ? JNI_ABORT : 0); if (decoded < 0) throwOpus(env, "opus_decode_float", decoded); return decoded; diff --git a/android/audio/src/main/java/com/ts3client/android/audio/NativeOpus.java b/android/audio/src/main/java/com/ts3client/android/audio/NativeOpus.java index 0673198..5abbb56 100644 --- a/android/audio/src/main/java/com/ts3client/android/audio/NativeOpus.java +++ b/android/audio/src/main/java/com/ts3client/android/audio/NativeOpus.java @@ -68,12 +68,17 @@ public final class NativeOpus implements OpusCodec { @Override public int decode(byte[] packet, float[] out) { - return NativeOpus.decode(live(), packet, out, maxFrameSize); + return NativeOpus.decode(live(), packet, out, maxFrameSize, false); } @Override public int conceal(float[] out, int frameSize) { - return NativeOpus.decode(live(), null, out, Math.min(frameSize, maxFrameSize)); + return NativeOpus.decode(live(), null, out, Math.min(frameSize, maxFrameSize), false); + } + + @Override + public int recover(byte[] next, float[] out, int frameSize) { + return NativeOpus.decode(live(), next, out, Math.min(frameSize, maxFrameSize), true); } @Override @@ -104,7 +109,7 @@ public final class NativeOpus implements OpusCodec { private static native long decoderCreate(int sampleRate, int channels); - private static native int decode(long handle, byte[] packet, float[] out, int frameSize); + private static native int decode(long handle, byte[] packet, float[] out, int frameSize, boolean fec); private static native void decoderReset(long handle); diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/JitterBuffer.java b/ts3-client/core/src/main/java/com/ts3client/audio/JitterBuffer.java index 2b24758..05afcd7 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/JitterBuffer.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/JitterBuffer.java @@ -184,8 +184,13 @@ public final class JitterBuffer { this.subWindowSize = this.windowSize / SUB_WINDOWS; } - /** Sets the extra delay held on top of the measured optimum. */ + /** + * Sets the extra delay held on top of the measured optimum. The timings already recorded + * are re-based onto it, so the delay moves at the next {@link #tick} rather than once the + * estimate has turned over. + */ public void setMargin(int margin) { + shiftTimings(bufferMargin - margin); this.bufferMargin = margin; } @@ -478,6 +483,12 @@ public final class JitterBuffer { shiftTimings(-optimal); pointerTimestamp += optimal; if (optimal < 0) interpolationRequested = -optimal; + // Held packets measure their slack against when they arrived, so move that with the + // timeline too. libspeex does not, which makes a delay jump of several frames count + // twice: every packet already waiting reports the old shortfall again on playout. + for (int i = 0; i < MAX_PACKETS; i++) { + if (data[i] != null && arrival[i] != 0) arrival[i] += optimal; + } } /** diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/StreamingVoiceOutput.java b/ts3-client/core/src/main/java/com/ts3client/audio/StreamingVoiceOutput.java index 53deb65..5a25131 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/StreamingVoiceOutput.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/StreamingVoiceOutput.java @@ -47,6 +47,8 @@ public final class StreamingVoiceOutput implements VoiceOutput { private volatile String outputDevice; private volatile double masterVolume = 1.0; + /** Extra playback delay, in jitter buffer ticks. */ + private volatile int extraDelayTicks; private volatile boolean deafened = false; /** Notified (clientId, talking) on an audio thread when a speaker starts/stops. */ @@ -149,6 +151,13 @@ public final class StreamingVoiceOutput implements VoiceOutput { this.outputDevice = device == null ? "" : device; } + @Override + public void setReceiveBuffer(int ms) { + int frames = Math.max(0, ms) * (VoiceFormat.SAMPLE_RATE / 1000) / VoiceFormat.FRAME_SIZE; + extraDelayTicks = frames * JitterBuffer.TEAMSPEAK_TICKS_PER_FRAME; + for (VoiceStream s : streams.values()) s.setExtraDelay(extraDelayTicks); + } + /** Entry point wired into {@code client.setVoiceHandler(...)}. */ @Override public void handleVoice(PacketBody0Voice voice) { @@ -168,6 +177,7 @@ public final class StreamingVoiceOutput implements VoiceOutput { VoiceStream stream = streams.computeIfAbsent(clientId, id -> { VoiceStream created = new VoiceStream(id, opus, this::fireTalk); created.setTap(voiceTap); + created.setExtraDelay(extraDelayTicks); return created; }); if (stream.offer(packetId, codec, data)) playout.started(stream); diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceOutput.java b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceOutput.java index e80be8c..d6bfd35 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceOutput.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceOutput.java @@ -34,6 +34,12 @@ public interface VoiceOutput { void setOutputDevice(String device); + /** + * Delays every speaker by {@code ms} more than the network calls for, trading latency for + * fewer dropouts on a bad connection. + */ + void setReceiveBuffer(int ms); + void removeClient(int clientId); void shutdown(); diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceReceiveStats.java b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceReceiveStats.java index 70a9f77..847d7f6 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceReceiveStats.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceReceiveStats.java @@ -6,23 +6,25 @@ package com.ts3client.audio; * * @param received voice packets that arrived, copies included * @param played frames played from a packet - * @param missed frames concealed because their packet was not there when due + * @param missed frames whose packet was not there when due + * @param recovered of those, frames rebuilt from the error correction copy in the next packet + * rather than concealed * @param outOfOrder packets that arrived after a later one * @param late packets that arrived after their frame was concealed, so were dropped; * each one is also among {@code missed} * @param duplicates copies of a packet already received, discarded */ -public record VoiceReceiveStats(long received, long played, long missed, long outOfOrder, long late, +public record VoiceReceiveStats(long received, long played, long missed, long recovered, long outOfOrder, long late, long duplicates) { - public static final VoiceReceiveStats NONE = new VoiceReceiveStats(0, 0, 0, 0, 0, 0); + public static final VoiceReceiveStats NONE = new VoiceReceiveStats(0, 0, 0, 0, 0, 0, 0); public VoiceReceiveStats plus(VoiceReceiveStats o) { return new VoiceReceiveStats(received + o.received, played + o.played, missed + o.missed, - outOfOrder + o.outOfOrder, late + o.late, duplicates + o.duplicates); + recovered + o.recovered, outOfOrder + o.outOfOrder, late + o.late, duplicates + o.duplicates); } - /** Frames that were due, whether played or concealed. */ + /** Frames that were due, whether played or missed. */ public long due() { return played + missed; } diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java index be1c83b..d0e9b80 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java @@ -20,6 +20,11 @@ import java.util.function.BiConsumer; * position is remembered, the burst ends once playback reaches it, and its slot is played as * silence rather than concealed, since nothing was lost there. * + *

A frame that did go missing is rebuilt from the error correction copy in the packet after + * it, when that one is already held; only otherwise is it concealed. An + * {@linkplain #setExtraDelay extra delay} makes that, and late packets, more likely to be in + * time. + * *

The stream keeps no clock of its own: whoever pulls sets the pace, one call per * 20 ms frame. {@link #offer} and {@link #stop} may be called from any thread; pulls * must come from one thread at a time. @@ -46,7 +51,7 @@ final class VoiceStream { private static final int SEEN_WINDOW = 1024; /** What a pull plays, decided under the lock and produced outside it. */ - private enum Source { PACKET, CONCEAL, SILENCE, SURPLUS } + private enum Source { PACKET, RECOVER, CONCEAL, SILENCE, SURPLUS } final int clientId; private final OpusCodec opus; @@ -67,6 +72,10 @@ final class VoiceStream { private int surplusFrames; /** Frames of concealment still owed to a delay increase. */ private int concealOwed; + /** Ticks of delay held on top of what the jitter buffer measures it needs. */ + private int extraTicks; + /** Frames of silence still to play before a fresh timeline starts, building up {@link #extraTicks}. */ + private int leadInFrames; /** Set when the timeline was dropped; the decoder follows on the next pull. */ private boolean resetDecoder; /** Recently received sequence numbers, each at {@code seq % SEEN_WINDOW}. */ @@ -75,6 +84,7 @@ final class VoiceStream { private long received; private long played; private long missed; + private long recovered; private long outOfOrder; private long late; private long duplicates; @@ -107,6 +117,17 @@ final class VoiceStream { this.tap = tap; } + /** + * Holds {@code ticks} more delay than the network calls for, for a better chance that + * late packets and those carrying a lost one's error correction copy arrive in time. + */ + void setExtraDelay(int ticks) { + synchronized (lock) { + extraTicks = Math.max(0, ticks); + jitter.setMargin(FRAME_TICKS + extraTicks); + } + } + private static int channelsFor(CodecType codec) { return codec == CodecType.OPUS_MUSIC ? 2 : 1; } @@ -154,6 +175,8 @@ final class VoiceStream { playing = true; started = true; emptyPulls = 0; + // The buffer would otherwise reach the extra delay by concealing frames mid-speech. + if (fresh) leadInFrames = extraTicks / FRAME_TICKS; } } markTalking(true); @@ -180,7 +203,10 @@ final class VoiceStream { resetDecoder = false; packetChannels = channels; - if (surplusFrames > 0 || concealOwed > 0) { + if (leadInFrames > 0) { + leadInFrames--; + source = Source.SILENCE; + } else if (surplusFrames > 0 || concealOwed > 0) { // Still playing out what the last get() handed over. if (surplusFrames > 0) { source = Source.SURPLUS; @@ -209,8 +235,14 @@ final class VoiceStream { if (isStopMark(jitter.timestamp())) { source = Source.SILENCE; } else { - source = Source.CONCEAL; missed++; + packet = jitter.nextPayload(); + if (packet != null) { + source = Source.RECOVER; + recovered++; + } else { + source = Source.CONCEAL; + } } } default -> { @@ -258,6 +290,11 @@ final class VoiceStream { play(pcm, 0, frames, decoderChannels, out, outChannels, gain); return frames; } + case RECOVER -> { + int frames = decoderFor(decoderChannels == 0 ? packetChannels : decoderChannels).recover(packet, pcm, FRAME); + play(pcm, 0, frames, decoderChannels, out, outChannels, gain); + return frames; + } default -> { int frames = decoderFor(packetChannels).decode(packet, pcm); int played = Math.min(frames, FRAME); @@ -372,7 +409,7 @@ final class VoiceStream { /** How this speaker's voice has been coming in since the stream was created. */ VoiceReceiveStats stats() { synchronized (lock) { - return new VoiceReceiveStats(received, played, missed, outOfOrder, late, duplicates); + return new VoiceReceiveStats(received, played, missed, recovered, outOfOrder, late, duplicates); } } @@ -415,6 +452,7 @@ final class VoiceStream { playing = false; surplusFrames = 0; concealOwed = 0; + leadInFrames = 0; emptyPulls = 0; } diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/opus/OpusDecoder.java b/ts3-client/core/src/main/java/com/ts3client/audio/opus/OpusDecoder.java index f5414f2..b2c4e77 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/opus/OpusDecoder.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/opus/OpusDecoder.java @@ -12,6 +12,14 @@ public interface OpusDecoder extends AutoCloseable { */ int conceal(float[] out, int frameSize); + /** + * Rebuilds {@code frameSize} samples per channel for a lost packet from the forward error + * correction copy carried in {@code next}, the packet that followed it. Falls back to + * concealment by itself when {@code next} carries none. {@code next} is then decoded as + * usual. + */ + int recover(byte[] next, float[] out, int frameSize); + /** Forgets the stream's state, as at the start of a new talk burst. */ void reset(); diff --git a/ts3-client/core/src/main/java/com/ts3client/config/Settings.java b/ts3-client/core/src/main/java/com/ts3client/config/Settings.java index 5080aca..1ee48db 100644 --- a/ts3-client/core/src/main/java/com/ts3client/config/Settings.java +++ b/ts3-client/core/src/main/java/com/ts3client/config/Settings.java @@ -133,6 +133,13 @@ public final class Settings { /** How many times every voice packet is sent again. */ public int redundancyCopies = 1; public static final int MAX_REDUNDANCY_COPIES = 4; + /** + * Milliseconds of playback delay held on top of what the jitter buffer measures it needs, + * so late packets and error correction copies still make it on a bad connection. + */ + public int receiveBuffer = 0; + /** The {@link #receiveBuffer} choices the settings offer; whole 20 ms voice frames. */ + public static final int[] RECEIVE_BUFFERS = {0, 20, 40, 60, 80, 120, 200, 300, 500}; /** Master playback gain, 0..1 (may exceed 1 for boost up to 2). */ public double outputVolume = 1.0; /** @@ -291,6 +298,7 @@ public final class Settings { music = parseB(props.getProperty("music"), music); redundancyDelay = parseI(props.getProperty("redundancyDelay"), redundancyDelay); redundancyCopies = parseI(props.getProperty("redundancyCopies"), redundancyCopies); + receiveBuffer = parseI(props.getProperty("receiveBuffer"), receiveBuffer); outputVolume = parseD(props.getProperty("outputVolume"), outputVolume); masterVolume = parseD(props.getProperty("masterVolume"), masterVolume); inputVolume = parseD(props.getProperty("inputVolume"), inputVolume); @@ -351,6 +359,7 @@ public final class Settings { props.setProperty("music", Boolean.toString(music)); props.setProperty("redundancyDelay", Integer.toString(redundancyDelay)); props.setProperty("redundancyCopies", Integer.toString(redundancyCopies)); + props.setProperty("receiveBuffer", Integer.toString(receiveBuffer)); props.setProperty("outputVolume", Double.toString(outputVolume)); props.setProperty("masterVolume", Double.toString(masterVolume)); props.setProperty("inputVolume", Double.toString(inputVolume)); diff --git a/ts3-client/core/src/main/java/com/ts3client/net/ConnectionStatsFormat.java b/ts3-client/core/src/main/java/com/ts3client/net/ConnectionStatsFormat.java index 817f730..791216f 100644 --- a/ts3-client/core/src/main/java/com/ts3client/net/ConnectionStatsFormat.java +++ b/ts3-client/core/src/main/java/com/ts3client/net/ConnectionStatsFormat.java @@ -121,6 +121,7 @@ public final class ConnectionStatsFormat { new Row("Received", known ? count(s.received()) : UNKNOWN), new Row("Played", known ? countOf(s.played(), s.due()) : UNKNOWN), new Row("Missed", known ? countOf(s.missed(), s.due()) : UNKNOWN), + new Row("Recovered (FEC)", known ? countOf(s.recovered(), s.missed()) : UNKNOWN), new Row("Out of order", known ? countOf(s.outOfOrder(), s.received()) : UNKNOWN), new Row("Dropped (too late)", known ? countOf(s.late(), s.received()) : UNKNOWN), new Row("Discarded (copies)", known ? countOf(s.duplicates(), s.received()) : UNKNOWN)); diff --git a/ts3-client/core/src/main/java/com/ts3client/net/TeamspeakConnection.java b/ts3-client/core/src/main/java/com/ts3client/net/TeamspeakConnection.java index 8598b06..250b648 100644 --- a/ts3-client/core/src/main/java/com/ts3client/net/TeamspeakConnection.java +++ b/ts3-client/core/src/main/java/com/ts3client/net/TeamspeakConnection.java @@ -285,6 +285,7 @@ public final class TeamspeakConnection implements TS3Listener { model.clear(); playback = audio.createOutput(settings); playback.setMasterVolume(settings.effectiveOutputVolume()); + playback.setReceiveBuffer(settings.receiveBuffer); playback.setTalkListener((clientId, talking) -> { ClientEntry c = model.getClient(clientId); if (c != null) c.talking = talking; diff --git a/ts3-client/core/src/test/java/com/ts3client/audio/FakeOpus.java b/ts3-client/core/src/test/java/com/ts3client/audio/FakeOpus.java index ab0139a..e8d8769 100644 --- a/ts3-client/core/src/test/java/com/ts3client/audio/FakeOpus.java +++ b/ts3-client/core/src/test/java/com/ts3client/audio/FakeOpus.java @@ -10,11 +10,12 @@ import java.util.List; /** * Stands in for libopus: a "packet" of n bytes decodes to n 20 ms frames holding the first byte - * divided by 100 in every sample. Concealment yields -1, so it is easy to spot. + * divided by 100 in every sample. Concealment yields -1 and recovery -2, so both are easy to spot. */ final class FakeOpus implements OpusCodec { static final float CONCEALED = -1f; + static final float RECOVERED = -2f; int resets; final List concealedFrameSizes = new ArrayList<>(); @@ -45,6 +46,12 @@ final class FakeOpus implements OpusCodec { return frameSize; } + @Override + public int recover(byte[] next, float[] out, int frameSize) { + Arrays.fill(out, 0, frameSize * channels, RECOVERED); + return frameSize; + } + @Override public void reset() { resets++; diff --git a/ts3-client/core/src/test/java/com/ts3client/audio/JitterBufferTest.java b/ts3-client/core/src/test/java/com/ts3client/audio/JitterBufferTest.java index 8d77387..9fdff01 100644 --- a/ts3-client/core/src/test/java/com/ts3client/audio/JitterBufferTest.java +++ b/ts3-client/core/src/test/java/com/ts3client/audio/JitterBufferTest.java @@ -51,6 +51,27 @@ class JitterBufferTest { assertTrue(real.size() >= 15, "most frames should have played: " + played); } + @Test + void aLargerMarginBuffersThatMuchMore() { + JitterBuffer b = buffer(); + int n = 0; + int lastPlayed = -1; + for (; n < 60; n++) { + put(b, n); + int tag = pull(b); + if (tag >= 0) lastPlayed = tag; + } + int before = n - 1 - lastPlayed; + + b.setMargin(b.margin() + 5 * SPAN); + for (; n < 120; n++) { + put(b, n); + int tag = pull(b); + if (tag >= 0) lastPlayed = tag; + } + assertEquals(before + 5, n - 1 - lastPlayed); + } + @Test void reordersPacketsThatArriveOutOfOrder() { JitterBuffer b = buffer(); diff --git a/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java b/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java index 97d1eff..149e6a4 100644 --- a/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java +++ b/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java @@ -59,15 +59,38 @@ class VoiceStreamTest { } @Test - void concealsALostPacket() { + void rebuildsALostPacketFromTheOneAfterIt() { offer(0, 1); offer(2, 3); assertEquals(0.01f, pull()); - assertEquals(FakeOpus.CONCEALED, pull()); + assertEquals(FakeOpus.RECOVERED, pull()); assertEquals(0.03f, pull()); + assertEquals(List.of(), opus.concealedFrameSizes); + } + + @Test + void concealsALostPacketWhenTheNextIsMissingToo() { + offer(0, 1); + offer(3, 4); + assertEquals(0.01f, pull()); + assertEquals(FakeOpus.CONCEALED, pull()); + assertEquals(FakeOpus.RECOVERED, pull()); + assertEquals(0.04f, pull()); assertEquals(List.of(VoiceFormat.FRAME_SIZE), opus.concealedFrameSizes); } + @Test + void anExtraDelayLeadsANewSpeakerInWithSilence() { + stream.setExtraDelay(2 * JitterBuffer.TEAMSPEAK_TICKS_PER_FRAME); + offer(0, 1); + assertEquals(0f, pull()); + offer(1, 2); + assertEquals(0f, pull()); + assertEquals(0.01f, pull()); + assertEquals(0.02f, pull()); + assertEquals(List.of(), opus.concealedFrameSizes); + } + @Test void dropsPacketsThatArriveTooLate() { offer(0, 1); @@ -100,7 +123,7 @@ class VoiceStreamTest { pull(); // 3 never arrives: concealed offer(3, 4); // too late, dropped offer(3, 4); // a copy of that - assertEquals(new VoiceReceiveStats(6, 3, 1, 1, 1, 2), stream.stats()); + assertEquals(new VoiceReceiveStats(6, 3, 1, 0, 1, 1, 2), stream.stats()); } @Test diff --git a/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/NativeOpusDecoder.java b/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/NativeOpusDecoder.java index 1954bc8..c1c5615 100644 --- a/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/NativeOpusDecoder.java +++ b/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/NativeOpusDecoder.java @@ -54,17 +54,26 @@ final class NativeOpusDecoder implements OpusDecoder { throw new IllegalArgumentException("packet too large: " + packet.length); } MemorySegment.copy(packet, 0, packetBuffer, ValueLayout.JAVA_BYTE, 0, packet.length); - return decode(packetBuffer, packet.length, out, frameSize); + return decode(packetBuffer, packet.length, out, frameSize, 0); } @Override public int conceal(float[] out, int frameSize) { - return decode(MemorySegment.NULL, 0, out, Math.min(frameSize, this.frameSize)); + return decode(MemorySegment.NULL, 0, out, Math.min(frameSize, this.frameSize), 0); } - private int decode(MemorySegment data, int len, float[] out, int maxSamples) { + @Override + public int recover(byte[] next, float[] out, int frameSize) { + if (next.length > MAX_PACKET_BYTES) { + throw new IllegalArgumentException("packet too large: " + next.length); + } + MemorySegment.copy(next, 0, packetBuffer, ValueLayout.JAVA_BYTE, 0, next.length); + return decode(packetBuffer, next.length, out, Math.min(frameSize, this.frameSize), 1); + } + + private int decode(MemorySegment data, int len, float[] out, int maxSamples, int fec) { if (closed) throw new IllegalStateException("decoder closed"); - int samples = Opus.decodeFloat(handle, data, len, pcmBuffer, maxSamples, 0); + int samples = Opus.decodeFloat(handle, data, len, pcmBuffer, maxSamples, fec); if (samples < 0) { throw new IllegalStateException("opus_decode_float failed: " + Opus.strerror(samples)); } diff --git a/ts3-client/swing/src/main/java/com/ts3client/ui/MainFrame.java b/ts3-client/swing/src/main/java/com/ts3client/ui/MainFrame.java index 9b2bf94..bef1f3c 100644 --- a/ts3-client/swing/src/main/java/com/ts3client/ui/MainFrame.java +++ b/ts3-client/swing/src/main/java/com/ts3client/ui/MainFrame.java @@ -964,13 +964,14 @@ public final class MainFrame extends JFrame implements ServerTabPane.Listener { dlg.setVisible(true); } - /** Master volume / output device are global, so push them to every open connection. */ + /** Master volume, output device and receive buffer are global, so push them to every open connection. */ private void applyOutputSettingsToAllTabs() { soundPlayer.setOutputDevice(settings.outputDevice); for (ServerTab tab : tabs) { if (tab.connection().getPlayback() == null) continue; tab.connection().getPlayback().setMasterVolume(settings.effectiveOutputVolume()); tab.connection().getPlayback().setOutputDevice(settings.outputDevice); + tab.connection().getPlayback().setReceiveBuffer(settings.receiveBuffer); } } diff --git a/ts3-client/swing/src/main/java/com/ts3client/ui/NetworkPanel.java b/ts3-client/swing/src/main/java/com/ts3client/ui/NetworkPanel.java index 3dc66d7..c718859 100644 --- a/ts3-client/swing/src/main/java/com/ts3client/ui/NetworkPanel.java +++ b/ts3-client/swing/src/main/java/com/ts3client/ui/NetworkPanel.java @@ -11,8 +11,8 @@ import java.awt.GridBagConstraints; /** * Options page for what voice costs and survives on the wire: the encoder's bitrate and - * error correction, and duplicate voice packets. Changes preview live on the connected - * microphone. + * error correction, duplicate voice packets, and how long incoming voice is buffered. + * Transmit changes preview live on the connected microphone. */ final class NetworkPanel extends FormPanel { @@ -26,6 +26,7 @@ final class NetworkPanel extends FormPanel { private final JCheckBox fecCheck; private final JComboBox redundancyCombo; private final JComboBox copiesCombo; + private final JComboBox receiveBufferCombo; /** @param pushEncoder puts the encoder settings on screen, across pages, into effect */ NetworkPanel(Settings settings, VoiceInput liveMic, Runnable pushEncoder) { @@ -45,13 +46,8 @@ final class NetworkPanel extends FormPanel { vbrCheck.addActionListener(e -> pushEncoder.run()); fecCheck.addActionListener(e -> pushEncoder.run()); - String[] delays = new String[Settings.REDUNDANCY_DELAYS.length]; - for (int i = 0; i < delays.length; i++) { - int ms = Settings.REDUNDANCY_DELAYS[i]; - delays[i] = ms == 0 ? "Off" : ms + " ms"; - } - redundancyCombo = new JComboBox<>(delays); - redundancyCombo.setSelectedIndex(nearestDelay(settings.redundancyDelay)); + redundancyCombo = new JComboBox<>(delayLabels(Settings.REDUNDANCY_DELAYS)); + redundancyCombo.setSelectedIndex(nearest(Settings.REDUNDANCY_DELAYS, settings.redundancyDelay)); Integer[] copies = new Integer[Settings.MAX_REDUNDANCY_COPIES]; for (int i = 0; i < copies.length; i++) copies[i] = i + 1; copiesCombo = new JComboBox<>(copies); @@ -63,6 +59,9 @@ final class NetworkPanel extends FormPanel { }); copiesCombo.addActionListener(e -> pushRedundancy()); + receiveBufferCombo = new JComboBox<>(delayLabels(Settings.RECEIVE_BUFFERS)); + receiveBufferCombo.setSelectedIndex(nearest(Settings.RECEIVE_BUFFERS, settings.receiveBuffer)); + int row = 0; addSection(this, c, row++, "Encoder"); addRow(this, c, row++, new JLabel("Bitrate:"), sliderWithLabel(bitrateSlider, bitrateLabel, 70)); @@ -71,6 +70,8 @@ final class NetworkPanel extends FormPanel { addSection(this, c, row++, "Redundancy"); addRow(this, c, row++, new JLabel("Delay:"), redundancyCombo); addRow(this, c, row++, new JLabel("Copies:"), copiesCombo); + addSection(this, c, row++, "Receive"); + addRow(this, c, row++, new JLabel("Extra buffering:"), receiveBufferCombo); addFiller(this, c, row); } @@ -82,10 +83,16 @@ final class NetworkPanel extends FormPanel { return (Integer) copiesCombo.getSelectedItem(); } - private static int nearestDelay(int ms) { + private static String[] delayLabels(int[] choices) { + String[] labels = new String[choices.length]; + for (int i = 0; i < labels.length; i++) labels[i] = choices[i] == 0 ? "Off" : choices[i] + " ms"; + return labels; + } + + private static int nearest(int[] choices, int ms) { int best = 0; - for (int i = 1; i < Settings.REDUNDANCY_DELAYS.length; i++) { - if (Math.abs(Settings.REDUNDANCY_DELAYS[i] - ms) < Math.abs(Settings.REDUNDANCY_DELAYS[best] - ms)) best = i; + for (int i = 1; i < choices.length; i++) { + if (Math.abs(choices[i] - ms) < Math.abs(choices[best] - ms)) best = i; } return best; } @@ -101,5 +108,6 @@ final class NetworkPanel extends FormPanel { target.fec = fecCheck.isSelected(); target.redundancyDelay = selectedDelay(); target.redundancyCopies = selectedCopies(); + target.receiveBuffer = Settings.RECEIVE_BUFFERS[receiveBufferCombo.getSelectedIndex()]; } }