Voice: stop clipping the start of words from native talkers

The native client keeps numbering voice packets through silence, so a new
word arrives far ahead of where playback stopped. Playback concealed its
way towards it and the jitter buffer's loss reset threw away the held
start of the word. Once the talker has let go, playback now skips to the
next burst, keeping the delay it held, and starts a fresh Opus decoder as
the talker's encoder does.

Also: a late copy of a passed packet no longer keeps the buffer looking
non-empty, and a burst whose stop was lost rewinds its concealment so the
next burst does not land behind playback.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-01 20:20:09 +00:00
co-authored by Claude Opus 5.5
parent 42870a702c
commit f6593527be
3 changed files with 168 additions and 14 deletions
@@ -207,13 +207,53 @@ public final class JitterBuffer {
return total; return total;
} }
/** True when no packet is held and playback has nothing left to drain. */ /**
* True when playback has nothing left to drain. A packet that arrived after playback
* passed it may still be held, for the timing it reported, but it can never be played.
*/
public boolean isEmpty() { public boolean isEmpty() {
if (resetState) {
for (int i = 0; i < MAX_PACKETS; i++) { for (int i = 0; i < MAX_PACKETS; i++) {
if (data[i] != null) return false; if (data[i] != null) return false;
} }
return true; return true;
} }
for (int i = 0; i < MAX_PACKETS; i++) {
if (data[i] != null && gt(timestamp[i] + span[i], pointerTimestamp)) return false;
}
return true;
}
/** Timestamp of the earliest packet still to be played; only meaningful when not {@link #isEmpty}. */
public int earliest() {
boolean found = false;
int earliest = pointerTimestamp;
for (int i = 0; i < MAX_PACKETS; i++) {
if (data[i] == null || (!resetState && !gt(timestamp[i] + span[i], pointerTimestamp))) continue;
if (!found || lt(timestamp[i], earliest)) {
earliest = timestamp[i];
found = true;
}
}
return earliest;
}
/**
* Moves playback to {@code timestamp}, keeping the delay estimate: held packets measure
* their slack against the moved timeline. Does nothing before the first packet has
* anchored the timeline.
*/
public void seek(int timestamp) {
if (resetState) return;
int delta = timestamp - pointerTimestamp;
for (int i = 0; i < MAX_PACKETS; i++) {
if (data[i] != null && arrival[i] != 0) arrival[i] += delta;
}
pointerTimestamp = timestamp;
nextStop = timestamp;
buffered = 0;
lostCount = 0;
}
/** /**
* Drops every held packet and forgets both the timeline and the network estimate. * Drops every held packet and forgets both the timeline and the network estimate.
@@ -10,11 +10,11 @@ import java.util.function.BiConsumer;
* One speaker's incoming voice: packets go into an adaptive {@link JitterBuffer} as they * One speaker's incoming voice: packets go into an adaptive {@link JitterBuffer} as they
* arrive, and every {@link #pull} plays out the next 20&nbsp;ms, concealing what went missing. * arrive, and every {@link #pull} plays out the next 20&nbsp;ms, concealing what went missing.
* *
* <p>The buffer is driven the way the official client drives it. A speaker has one timeline, * <p>A speaker has one timeline, {@code unwrap16(packetId) * 60} ticks, kept across talk
* {@code unwrap16(packetId) * 60} ticks, which is never re-anchored between talk bursts: the * bursts so the delay the buffer has learned carries over. The official client keeps
* sender stops numbering when it stops talking and playback stops at the same point, so the * numbering while its user is silent, so a burst usually starts well ahead of where the last
* next burst lands where the last one left off and the delay the buffer has learned carries * one stopped; once the talker has let go, playback skips to the next burst, holding the
* over. Only a long silence drops the timeline. * same delay. Only a long silence drops the timeline.
* *
* <p>An empty packet is the talker letting go of the key. It never enters the buffer; its * <p>An empty packet is the talker letting go of the key. It never enters the buffer; its
* position is remembered, the burst ends once playback reaches it, and its slot is played as * position is remembered, the burst ends once playback reaches it, and its slot is played as
@@ -64,6 +64,14 @@ final class VoiceStream {
private int lastSeq; private int lastSeq;
private int[] stopMarks = new int[8]; private int[] stopMarks = new int[8];
private int stopMarkCount; private int stopMarkCount;
/** Where the audio last played from a packet ends; the talker resumes from there. */
private int heard;
private boolean heardAny;
/** How far playback trailed the talker when they let go; the next burst is held back as much. */
private int stopLag = FRAME_TICKS;
/** Slots before this one are the silence ahead of a burst skipped to, not losses. */
private int burstStart;
private boolean skippedTo;
/** Channels of the latest packet: TeamSpeak streams {@code OPUS_MUSIC} in stereo. */ /** Channels of the latest packet: TeamSpeak streams {@code OPUS_MUSIC} in stereo. */
private int channels = 1; private int channels = 1;
private boolean playing; private boolean playing;
@@ -76,7 +84,7 @@ final class VoiceStream {
private int extraTicks; private int extraTicks;
/** Frames of silence still to play before a fresh timeline starts, building up {@link #extraTicks}. */ /** Frames of silence still to play before a fresh timeline starts, building up {@link #extraTicks}. */
private int leadInFrames; private int leadInFrames;
/** Set when the timeline was dropped; the decoder follows on the next pull. */ /** Set when a burst ended or the timeline was dropped: the talker's encoder starts afresh. */
private boolean resetDecoder; private boolean resetDecoder;
/** Recently received sequence numbers, each at {@code seq % SEEN_WINDOW}. */ /** Recently received sequence numbers, each at {@code seq % SEEN_WINDOW}. */
private final int[] seen = new int[SEEN_WINDOW]; private final int[] seen = new int[SEEN_WINDOW];
@@ -150,6 +158,9 @@ final class VoiceStream {
jitter.reset(); jitter.reset();
epoch = EPOCH_UNSET; epoch = EPOCH_UNSET;
stopMarkCount = 0; stopMarkCount = 0;
heardAny = false;
skippedTo = false;
stopLag = FRAME_TICKS;
resetDecoder = true; resetDecoder = true;
forgetSeen(); forgetSeen();
} }
@@ -158,6 +169,7 @@ final class VoiceStream {
int ticks = seq * FRAME_TICKS; int ticks = seq * FRAME_TICKS;
if (end) { if (end) {
markStop(ticks); markStop(ticks);
if (playing) stopLag = Math.max(jitter.margin(), ticks - jitter.position());
return false; return false;
} }
boolean behind = !fresh && ticks - jitter.position() < 0; boolean behind = !fresh && ticks - jitter.position() < 0;
@@ -199,8 +211,6 @@ final class VoiceStream {
boolean reset; boolean reset;
synchronized (lock) { synchronized (lock) {
if (!playing) return IDLE; if (!playing) return IDLE;
reset = resetDecoder;
resetDecoder = false;
packetChannels = channels; packetChannels = channels;
if (leadInFrames > 0) { if (leadInFrames > 0) {
@@ -218,6 +228,7 @@ final class VoiceStream {
} }
jitter.remainingSpan(remainingTicks()); jitter.remainingSpan(remainingTicks());
} else { } else {
skipToNextBurst();
boolean empty = jitter.isEmpty(); boolean empty = jitter.isEmpty();
emptyPulls = empty ? emptyPulls + 1 : 0; emptyPulls = empty ? emptyPulls + 1 : 0;
if (empty && (stoppedAt(jitter.position()) || emptyPulls >= EMPTY_PULLS_TO_END)) { if (empty && (stoppedAt(jitter.position()) || emptyPulls >= EMPTY_PULLS_TO_END)) {
@@ -230,9 +241,12 @@ final class VoiceStream {
source = Source.PACKET; source = Source.PACKET;
packet = jitter.payload(); packet = jitter.payload();
played++; played++;
skippedTo = false;
heard(jitter.position());
} }
case MISSING -> { case MISSING -> {
if (isStopMark(jitter.timestamp())) { if (isStopMark(jitter.timestamp())
|| (skippedTo && jitter.timestamp() - burstStart < 0)) {
source = Source.SILENCE; source = Source.SILENCE;
} else { } else {
missed++; missed++;
@@ -240,6 +254,7 @@ final class VoiceStream {
if (packet != null) { if (packet != null) {
source = Source.RECOVER; source = Source.RECOVER;
recovered++; recovered++;
heard(jitter.position());
} else { } else {
source = Source.CONCEAL; source = Source.CONCEAL;
} }
@@ -254,6 +269,8 @@ final class VoiceStream {
jitter.tick(); jitter.tick();
} }
} }
reset = source != null && resetDecoder;
if (reset) resetDecoder = false;
} }
if (source == null) { if (source == null) {
markTalking(false); markTalking(false);
@@ -431,11 +448,36 @@ final class VoiceStream {
return false; return false;
} }
/** Whether the talker let go at or shortly before {@code position}, so the burst is spoken out. */ /**
* Once the talker has let go, moves playback up to the next burst they started, which may
* be numbered far ahead, trailing it by the delay held when they stopped. Call under the lock.
*/
private void skipToNextBurst() {
int position = jitter.position();
if (jitter.isEmpty() || !stoppedAt(position)) return;
int next = jitter.earliest();
int target = next - stopLag;
if (target - position <= 0) return;
jitter.seek(target);
burstStart = next;
skippedTo = true;
resetDecoder = true;
}
private void heard(int position) {
heard = position;
heardAny = true;
}
/**
* Whether the talker let go between the last audio heard and {@code position}, so the
* burst is spoken out. An earlier burst's stop does not count.
*/
private boolean stoppedAt(int position) { private boolean stoppedAt(int position) {
for (int i = 0; i < stopMarkCount; i++) { for (int i = 0; i < stopMarkCount; i++) {
int behind = position - stopMarks[i]; int behind = position - stopMarks[i];
if (behind >= 0 && behind < STOP_MARK_WINDOW_TICKS) return true; if (behind < 0 || behind >= STOP_MARK_WINDOW_TICKS) continue;
if (!heardAny || stopMarks[i] - heard >= 0) return true;
} }
return false; return false;
} }
@@ -449,7 +491,14 @@ final class VoiceStream {
} }
private void stopLocked() { private void stopLocked() {
// Concealment past the last packet was a guess that the talker was still going. Take
// it back, or the next burst, numbered on from that packet, lands behind playback.
if (heardAny && jitter.position() - heard > 0) {
if (!stoppedAt(jitter.position())) markStop(heard);
jitter.seek(heard);
}
playing = false; playing = false;
resetDecoder = true;
surplusFrames = 0; surplusFrames = 0;
concealOwed = 0; concealOwed = 0;
leadInFrames = 0; leadInFrames = 0;
@@ -146,7 +146,7 @@ class VoiceStreamTest {
assertTrue(offer(2, 2), "the next packet starts a new burst"); assertTrue(offer(2, 2), "the next packet starts a new burst");
assertEquals(0f, pull(), "the stop's slot is the talker's own silence, not a loss"); assertEquals(0f, pull(), "the stop's slot is the talker's own silence, not a loss");
assertEquals(0.02f, pull()); assertEquals(0.02f, pull());
assertEquals(0, opus.resets, "the decoder keeps its state between bursts"); assertEquals(1, opus.resets, "each burst starts a fresh decoder, as the talker's encoder does");
assertEquals(List.of(), opus.concealedFrameSizes); assertEquals(List.of(), opus.concealedFrameSizes);
} }
@@ -160,6 +160,71 @@ class VoiceStreamTest {
assertEquals(List.of(true, false), talk); assertEquals(List.of(true, false), talk);
} }
/** The official client keeps numbering through silence, so its next burst starts far ahead. */
@Test
void skipsToABurstNumberedAheadOfWhereTheLastStopped() {
offer(0, 1);
offer(1, 2);
offerEnd(2);
pull();
pull();
assertEquals(VoiceStream.IDLE, stream.pull(out, 1, 1.0));
assertTrue(offer(60, 3));
List<Float> heard = new ArrayList<>();
for (int i = 0; i < 8; i++) {
offer(61 + i, 4 + i);
heard.add(pull());
}
assertEquals(0.03f, heard.stream().filter(v -> v != 0f).findFirst().orElseThrow(),
"the burst plays from its first packet: " + heard);
assertTrue(heard.indexOf(0.03f) <= 2, "after at most the delay held before: " + heard);
assertEquals(List.of(), opus.concealedFrameSizes);
assertEquals(1, opus.resets);
}
@Test
void aBurstThatArrivesBeforeTheLastDrainedStartsAFreshDecoder() {
offer(0, 1);
offer(1, 2);
offerEnd(2);
pull();
offer(60, 3);
offer(61, 4);
List<Float> heard = new ArrayList<>();
for (int i = 0; i < 6; i++) heard.add(pull());
assertTrue(heard.contains(0.02f) && heard.contains(0.03f), "both bursts play in full: " + heard);
assertEquals(1, opus.resets);
}
@Test
void theBurstAfterALostStopPlaysFromItsFirstPacket() {
offer(0, 1);
offer(1, 2);
while (stream.pull(out, 1, 1.0) != VoiceStream.IDLE) {
// plays both, then conceals until it gives up on the stop
}
assertTrue(offer(2, 3), "the talker resumes where the last burst's audio ended");
assertEquals(0.03f, pull());
}
@Test
void aLateCopyOfTheLastPacketDoesNotHoldTheBurstOpen() {
offer(0, 1);
offer(1, 2);
offerEnd(2);
pull();
pull();
offer(1, 2); // a copy, after playback passed it
assertEquals(VoiceStream.IDLE, stream.pull(out, 1, 1.0));
assertTrue(offer(3, 4));
float sample = pull();
// The late copy taught the buffer to wait a frame longer, so the start may be padded.
for (int i = 0; i < 3 && sample <= 0; i++) sample = pull();
assertEquals(0.04f, sample);
}
@Test @Test
void playsALongPacketOneFrameAtATime() { void playsALongPacketOneFrameAtATime() {
stream.offer(0, CodecType.OPUS_VOICE, new byte[]{40, 0}); stream.offer(0, CodecType.OPUS_VOICE, new byte[]{40, 0});