Play incoming voice through TeamSpeak's adaptive jitter buffer

Each speaker's stream held a fixed 60 ms and gave up after 200 ms of
concealment, so a jittery connection concealed constantly and a stall
ended the burst. It now runs through a port of the libspeex jitter
buffer the official client links, configured and driven as it does:

- the delay adapts to the measured jitter, aiming for at most 1% of
  packets arriving late
- a speaker keeps one timeline, never re-anchored between talk bursts,
  so what the buffer learned carries over; only 5 s without packets
  drops it
- the talker's stop marker never enters the buffer; playback ends when
  it reaches it, and its slot is played as silence, not concealed

A pull now always hands out one 20 ms frame: what a longer packet or a
delay increase produces beyond that is played on the following pulls,
which Android's mixer used to cut off.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-25 10:46:08 +00:00
parent 7cb4f47a81
commit 0083dc8ad6
5 changed files with 1226 additions and 129 deletions

View File

@@ -9,7 +9,7 @@ import java.util.Arrays;
import java.util.List;
/**
* Stands in for libopus: a "packet" is one byte, decoded to a 20 ms frame holding that byte
* Stands in for libopus: a "packet" of n bytes decodes to n 20 ms frames holding the first byte
* divided by 100 in every sample. Concealment yields -1, so it is easy to spot.
*/
final class FakeOpus implements OpusCodec {
@@ -33,8 +33,9 @@ final class FakeOpus implements OpusCodec {
return new OpusDecoder() {
@Override
public int decode(byte[] packet, float[] out) {
Arrays.fill(out, 0, VoiceFormat.FRAME_SIZE * channels, packet[0] / 100f);
return VoiceFormat.FRAME_SIZE;
int frames = VoiceFormat.FRAME_SIZE * packet.length;
Arrays.fill(out, 0, frames * channels, packet[0] / 100f);
return frames;
}
@Override

View File

@@ -0,0 +1,353 @@
package com.ts3client.audio;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
class JitterBufferTest {
private static final int SPAN = JitterBuffer.TEAMSPEAK_TICKS_PER_FRAME;
private static JitterBuffer buffer() {
return JitterBuffer.forTeamSpeakVoice(SPAN);
}
/** Frame {@code n} of a stream, tagged so it can be identified on the way out. */
private static byte[] frame(int n) {
return new byte[]{(byte) n};
}
private static void put(JitterBuffer b, int n) {
b.put(frame(n), n * SPAN, SPAN);
}
/** Pulls one frame, returning its tag, or -1 for concealment. */
private static int pull(JitterBuffer b) {
JitterBuffer.Status status = b.get(SPAN);
int tag = status == JitterBuffer.Status.OK ? b.payload()[0] : -1;
b.tick();
return tag;
}
@Test
void playsAnUndisturbedStreamInOrder() {
JitterBuffer b = buffer();
List<Integer> played = new ArrayList<>();
for (int i = 0; i < 20; i++) {
put(b, i);
played.add(pull(b));
}
// The buffer holds some delay back, so the tail is still queued; nothing is reordered.
List<Integer> real = played.stream().filter(n -> n >= 0).toList();
assertEquals(real.stream().sorted().toList(), real);
assertTrue(real.size() >= 15, "most frames should have played: " + played);
}
@Test
void reordersPacketsThatArriveOutOfOrder() {
JitterBuffer b = buffer();
// Prime a delay so a swapped pair still lands in time.
for (int i = 0; i < 12; i++) {
put(b, i);
pull(b);
}
put(b, 13);
put(b, 12);
put(b, 14);
List<Integer> played = new ArrayList<>();
for (int i = 0; i < 6; i++) played.add(pull(b));
List<Integer> real = played.stream().filter(n -> n >= 0).toList();
assertEquals(real.stream().sorted().toList(), real, "played out of order: " + played);
assertTrue(real.contains(12) && real.contains(13) && real.contains(14), "lost a frame: " + played);
}
@Test
void reportsMissingFramesInsteadOfSkippingAhead() {
JitterBuffer b = buffer();
for (int i = 0; i < 30; i++) {
if (i != 20) put(b, i);
}
int concealed = 0;
int played = 0;
for (int i = 0; i < 30; i++) {
if (pull(b) < 0) concealed++;
else played++;
}
assertTrue(concealed >= 1, "the gap should have been concealed");
assertEquals(29, played, "every delivered frame should still be played");
}
@Test
void everyGetYieldsExactlyOneFrameOfAudio() {
JitterBuffer b = buffer();
Random random = new Random(7);
long ticks = 0;
for (int i = 0; i < 200; i++) {
if (random.nextInt(10) != 0) put(b, i);
b.get(SPAN);
assertEquals(0, b.span() % SPAN, "a partial frame cannot be rendered");
assertTrue(b.span() > 0);
ticks += b.span();
b.tick();
}
// Concealment for a growing delay may run long, but audio never stops or doubles up.
assertTrue(ticks >= 200L * SPAN, "output underran: " + ticks);
}
@Test
void growsTheDelayWhenArrivalsAreJittery() {
JitterBuffer b = buffer();
Random random = new Random(11);
// Deliver in bursts: frames pile up, then nothing arrives for a while.
int next = 0;
int concealedEarly = 0;
int concealedLate = 0;
for (int i = 0; i < 400; i++) {
if (i % 5 == 0) {
int burst = 3 + random.nextInt(4);
for (int j = 0; j < burst && next < 400; j++) put(b, next++);
}
boolean concealed = pull(b) < 0;
if (i < 100) concealedEarly += concealed ? 1 : 0;
else if (i >= 300) concealedLate += concealed ? 1 : 0;
}
assertTrue(concealedLate < concealedEarly,
"the buffer should settle: " + concealedEarly + " -> " + concealedLate);
}
@Test
void staysStableAcrossTimestampWraparound() {
JitterBuffer b = buffer();
// Start just below the point where the tick counter overflows.
int base = Integer.MAX_VALUE - 5 * SPAN;
for (int i = 0; i < 40; i++) {
b.put(frame(i), base + i * SPAN, SPAN);
}
int played = 0;
for (int i = 0; i < 40; i++) {
if (b.get(SPAN) == JitterBuffer.Status.OK) {
assertNotNull(b.payload());
played++;
}
b.tick();
}
assertTrue(played >= 35, "wraparound lost frames: " + played);
}
@Test
void resetForgetsTheTimelineSoTheNextBurstResyncs() {
JitterBuffer b = buffer();
for (int i = 0; i < 10; i++) {
put(b, i);
pull(b);
}
b.reset();
assertTrue(b.isEmpty());
// A new burst numbered far away must still play, not be treated as hopelessly late.
b.put(frame(1), 500 * SPAN, SPAN);
assertEquals(JitterBuffer.Status.OK, b.get(SPAN));
assertEquals(1, b.payload()[0]);
}
@Test
void dropsPacketsThatArriveHopelesslyLate() {
JitterBuffer b = buffer();
for (int i = 0; i < 40; i++) {
put(b, i);
pull(b);
}
assertFalse(b.isEmpty() && b.bufferedSpan() != 0);
// A frame from far in the past has nowhere to go and must not stall the stream.
b.put(frame(99), 0, SPAN);
for (int i = 0; i < 5; i++) {
assertTrue(pull(b) != 99, "a hopelessly late frame was played");
}
}
@Test
void teamSpeakConfigurationMatchesTheShippedClient() {
JitterBuffer b = JitterBuffer.forTeamSpeakVoice(JitterBuffer.TEAMSPEAK_TICKS_PER_FRAME);
assertEquals(60, JitterBuffer.TEAMSPEAK_TICKS_PER_FRAME);
assertEquals(60, b.margin());
}
/** Plays until nothing is held, leaving the position just past the last real frame. */
private static void drain(JitterBuffer b) {
for (int i = 0; i < 100 && !b.isEmpty(); i++) pull(b);
}
@Test
void offersTheFollowingPacketToRebuildALostOne() {
JitterBuffer b = buffer();
for (int i = 0; i < 12; i++) put(b, i);
drain(b);
// The frame due now never arrives, but the one after it is already here,
// carrying the FEC copy that can rebuild it.
int due = b.position();
b.put(frame(13), due + SPAN, SPAN);
assertEquals(JitterBuffer.Status.MISSING, b.get(SPAN));
assertNotNull(b.nextPayload(), "the successor should be offered for FEC recovery");
assertEquals(13, b.nextPayload()[0]);
b.tick();
// And it is still there to be played normally afterwards.
assertEquals(13, pull(b));
}
@Test
void offersNothingWhenTheSuccessorIsAlsoLost() {
JitterBuffer b = buffer();
for (int i = 0; i < 12; i++) put(b, i);
drain(b);
// Two frames in a row are gone; FEC only reaches back one.
int due = b.position();
b.put(frame(15), due + 2 * SPAN, SPAN);
assertEquals(JitterBuffer.Status.MISSING, b.get(SPAN));
assertNull(b.nextPayload(), "a double loss cannot be rebuilt from FEC");
}
@Test
void aPauseInSpeechCostsNothingWhenTheClockStopsToo() {
JitterBuffer b = buffer();
for (int i = 0; i < 40; i++) {
put(b, i);
pull(b);
}
drain(b);
// Silence. No packets arrive and nothing is pulled, so the position stays put —
// which is what lets the sender's numbering still line up when speech resumes.
int paused = b.position();
assertEquals(paused, b.position());
b.put(frame(7), paused, SPAN);
assertEquals(JitterBuffer.Status.OK, b.get(SPAN), "the next burst should play, not be judged late");
assertEquals(7, b.payload()[0]);
}
@Test
void keepsItsEstimateAcrossAPause() {
JitterBuffer b = buffer();
Random random = new Random(5);
int next = 0;
for (int i = 0; i < 300; i++) {
if (i % 4 == 0) {
for (int j = 0, burst = 2 + random.nextInt(4); j < burst; j++) put(b, next++);
}
pull(b);
}
drain(b);
JitterBuffer fresh = buffer();
assertTrue(concealedInBurst(b, b.position() / SPAN) <= concealedInBurst(fresh, 1000),
"a buffer that survived the pause should conceal no more than a cold one");
}
/** Plays a short jittery burst through a buffer and counts the concealed frames. */
private static int concealedInBurst(JitterBuffer b, int firstFrame) {
Random random = new Random(9);
int concealed = 0;
int next = firstFrame;
for (int i = 0; i < 60; i++) {
if (i % 4 == 0) {
for (int j = 0, burst = 2 + random.nextInt(4); j < burst; j++) {
b.put(frame(next), next * SPAN, SPAN);
next++;
}
}
if (pull(b) < 0) concealed++;
}
return concealed;
}
@Test
void resetForgetsTheNetworkEstimateToo() {
JitterBuffer b = buffer();
for (int i = 0; i < 100; i++) {
put(b, i);
pull(b);
}
b.reset();
// With no timings recorded there is nothing to justify moving the delay.
assertTrue(b.isEmpty());
assertEquals(0, b.heldFrames(SPAN));
}
@Test
void heldFramesCountsWhatIsWaiting() {
JitterBuffer b = buffer();
assertEquals(0, b.heldFrames(SPAN));
put(b, 0);
put(b, 1);
put(b, 2);
assertEquals(3, b.heldFrames(SPAN));
}
@Test
void aShortUtteranceComesOutWhole() {
JitterBuffer b = buffer();
int played = 0;
// Five frames, one per frame period — a single short word.
for (int i = 0; i < 5; i++) {
put(b, i);
if (pull(b) >= 0) played++;
}
// Then whatever the delay is still holding back.
for (int i = 0; i < 20 && !b.isEmpty(); i++) {
if (pull(b) >= 0) played++;
}
assertEquals(5, played, "every frame of a short utterance must be heard");
}
@Test
void pullingOnlyWhenAPacketIsWaitingStarvesTheStream() {
// The buffer adapts its delay in tick(), which only runs on a pull. A consumer that
// skips the pull whenever the buffer happens to be empty pins the delay at zero —
// and at zero delay the buffer is empty much of the time, so it never recovers.
int periods = 300;
assertEquals(periods, emitted(true), "a frame per period keeps the device fed");
int gated = emitted(false);
assertTrue(gated < periods * 9 / 10,
"gating the pull on a non-empty buffer should visibly starve the device: " + gated);
}
/** Runs a jittery arrival schedule for 300 periods, counting frames actually emitted. */
private static int emitted(boolean pullEveryPeriod) {
JitterBuffer b = buffer();
Random random = new Random(13);
int next = 0;
int frames = 0;
for (int i = 0; i < 300; i++) {
// Arrivals clump and stall, as they do on a real connection.
if (random.nextInt(4) != 0) {
for (int j = 0, burst = random.nextInt(3); j < burst; j++) put(b, next++);
}
if (pullEveryPeriod || !b.isEmpty()) {
b.get(SPAN);
b.tick();
frames++;
}
}
return frames;
}
}

View File

@@ -5,6 +5,7 @@ import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -21,6 +22,10 @@ class VoiceStreamTest {
return stream.offer(packetId, CodecType.OPUS_VOICE, FakeOpus.packet(value));
}
private void offerEnd(int packetId) {
stream.offer(packetId, CodecType.OPUS_VOICE, new byte[0]);
}
/** Pulls one frame and returns its first sample, or NaN if nothing was played. */
private float pull() {
int frames = stream.pull(out, 1, 1.0);
@@ -28,13 +33,11 @@ class VoiceStreamTest {
}
@Test
void holdsBackTheStartOfABurstThenPlaysInOrder() {
void playsPacketsInOrderWhateverOrderTheyArriveIn() {
assertTrue(offer(10, 1));
assertFalse(offer(12, 3));
assertFalse(offer(11, 2));
assertEquals(Float.NaN, pull());
assertEquals(Float.NaN, pull());
assertEquals(0.01f, pull());
assertEquals(0.02f, pull());
assertEquals(0.03f, pull());
@@ -45,13 +48,10 @@ class VoiceStreamTest {
void concealsALostPacket() {
offer(0, 1);
offer(2, 3);
pull();
pull();
assertEquals(0.01f, pull());
assertEquals(FakeOpus.CONCEALED, pull());
assertEquals(0.03f, pull());
assertEquals(List.of(VoiceFormat.FRAME_SIZE), opus.concealedFrameSizes,
"a lost packet is concealed at the length of the one before, not the longest Opus frame");
assertEquals(List.of(VoiceFormat.FRAME_SIZE), opus.concealedFrameSizes);
}
@Test
@@ -60,38 +60,54 @@ class VoiceStreamTest {
offer(1, 2);
pull();
pull();
pull();
pull();
assertFalse(offer(0, 9));
assertEquals(FakeOpus.CONCEALED, pull());
}
@Test
void endPacketFinishesTheBurst() {
void theTalkersStopEndsTheBurst() {
offer(0, 1);
stream.offer(1, CodecType.OPUS_VOICE, new byte[0]);
pull();
pull();
pull();
offerEnd(1);
assertEquals(0.01f, pull());
assertEquals(VoiceStream.IDLE, stream.pull(out, 1, 1.0));
assertFalse(stream.isPlaying());
assertEquals(List.of(true, false), talk);
assertEquals(1, opus.resets);
assertTrue(offer(2, 1), "the next packet starts a new burst");
}
@Test
void givesUpAfterTooMuchConcealment() {
void theNextBurstCarriesOnWhereTheLastStopped() {
offer(0, 1);
offerEnd(1);
pull();
pull();
assertTrue(offer(2, 2), "the next packet starts a new burst");
assertEquals(0f, pull(), "the stop's slot is the talker's own silence, not a loss");
assertEquals(0.02f, pull());
assertEquals(0, opus.resets, "the decoder keeps its state between bursts");
assertEquals(List.of(), opus.concealedFrameSizes);
}
@Test
void givesUpOnABurstWhoseStopWasLost() {
offer(0, 1);
pull();
int concealed = 0;
while (stream.pull(out, 1, 1.0) != VoiceStream.IDLE) concealed++;
assertEquals(10, concealed);
assertTrue(concealed > 0 && concealed < 10, "concealed " + concealed + " frames");
assertEquals(List.of(true, false), talk);
}
@Test
void playsALongPacketOneFrameAtATime() {
stream.offer(0, CodecType.OPUS_VOICE, new byte[]{40, 0});
offer(1, 50);
assertEquals(VoiceFormat.FRAME_SIZE, stream.pull(out, 1, 1.0));
assertEquals(0.4f, out[0]);
assertEquals(VoiceFormat.FRAME_SIZE, stream.pull(out, 1, 1.0));
assertEquals(0.4f, out[0]);
assertEquals(0.5f, pull());
}
@Test
void stopFromAnotherThreadEndsTheBurst() {
offer(0, 1);
@@ -103,11 +119,37 @@ class VoiceStreamTest {
@Test
void spreadsMonoOverStereoWithGain() {
offer(0, 50);
stream.pull(out, 2, 0.5);
stream.pull(out, 2, 0.5);
assertEquals(VoiceFormat.FRAME_SIZE, stream.pull(out, 2, 0.5));
assertEquals(0.25f, out[0]);
assertEquals(0.25f, out[1]);
assertEquals(0.25f, out[2 * VoiceFormat.FRAME_SIZE - 1]);
}
@Test
void neverHandsOutMoreThanOneFrame() {
for (int i = 0; i < 50; i++) offer(i, 1);
for (int i = 0; i < 50; i++) assertTrue(stream.pull(out, 1, 1.0) <= VoiceFormat.FRAME_SIZE);
}
/**
* Packets sent every 20 ms arrive up to 60 ms late. A fixed delay short of that conceals
* constantly; the adaptive one should learn the spread and stop concealing.
*/
@Test
void learnsADelayThatRidesOutJitter() {
Random random = new Random(1);
int packets = 1500;
long[] arrival = new long[packets];
for (int i = 0; i < packets; i++) arrival[i] = i * 20L + random.nextInt(61);
int next = 0;
int concealedLate = 0;
for (long now = 0; now < packets * 20L; now += 20) {
for (; next < packets && arrival[next] <= now; next++) offer(next, 1);
// arrivals are not in order, but whatever is due by now has been offered
float sample = pull();
if (now >= packets * 10L && sample == FakeOpus.CONCEALED) concealedLate++;
}
assertTrue(concealedLate < packets / 2 / 50, "concealed " + concealedLate + " frames in the second half");
}
}