Voice: stop clipping the start of words from native talkers
The native client keeps numbering voice packets through silence, so a new word arrives far ahead of where playback stopped. Playback concealed its way towards it and the jitter buffer's loss reset threw away the held start of the word. Once the talker has let go, playback now skips to the next burst, keeping the delay it held, and starts a fresh Opus decoder as the talker's encoder does. Also: a late copy of a passed packet no longer keeps the buffer looking non-empty, and a burst whose stop was lost rewinds its concealment so the next burst does not land behind playback. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -207,14 +207,54 @@ public final class JitterBuffer {
|
||||
return total;
|
||||
}
|
||||
|
||||
/** True when no packet is held and playback has nothing left to drain. */
|
||||
/**
|
||||
* True when playback has nothing left to drain. A packet that arrived after playback
|
||||
* passed it may still be held, for the timing it reported, but it can never be played.
|
||||
*/
|
||||
public boolean isEmpty() {
|
||||
if (resetState) {
|
||||
for (int i = 0; i < MAX_PACKETS; i++) {
|
||||
if (data[i] != null) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
for (int i = 0; i < MAX_PACKETS; i++) {
|
||||
if (data[i] != null) return false;
|
||||
if (data[i] != null && gt(timestamp[i] + span[i], pointerTimestamp)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Timestamp of the earliest packet still to be played; only meaningful when not {@link #isEmpty}. */
|
||||
public int earliest() {
|
||||
boolean found = false;
|
||||
int earliest = pointerTimestamp;
|
||||
for (int i = 0; i < MAX_PACKETS; i++) {
|
||||
if (data[i] == null || (!resetState && !gt(timestamp[i] + span[i], pointerTimestamp))) continue;
|
||||
if (!found || lt(timestamp[i], earliest)) {
|
||||
earliest = timestamp[i];
|
||||
found = true;
|
||||
}
|
||||
}
|
||||
return earliest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves playback to {@code timestamp}, keeping the delay estimate: held packets measure
|
||||
* their slack against the moved timeline. Does nothing before the first packet has
|
||||
* anchored the timeline.
|
||||
*/
|
||||
public void seek(int timestamp) {
|
||||
if (resetState) return;
|
||||
int delta = timestamp - pointerTimestamp;
|
||||
for (int i = 0; i < MAX_PACKETS; i++) {
|
||||
if (data[i] != null && arrival[i] != 0) arrival[i] += delta;
|
||||
}
|
||||
pointerTimestamp = timestamp;
|
||||
nextStop = timestamp;
|
||||
buffered = 0;
|
||||
lostCount = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drops every held packet and forgets both the timeline and the network estimate.
|
||||
*
|
||||
|
||||
@@ -10,11 +10,11 @@ import java.util.function.BiConsumer;
|
||||
* One speaker's incoming voice: packets go into an adaptive {@link JitterBuffer} as they
|
||||
* arrive, and every {@link #pull} plays out the next 20 ms, concealing what went missing.
|
||||
*
|
||||
* <p>The buffer is driven the way the official client drives it. A speaker has one timeline,
|
||||
* {@code unwrap16(packetId) * 60} ticks, which is never re-anchored between talk bursts: the
|
||||
* sender stops numbering when it stops talking and playback stops at the same point, so the
|
||||
* next burst lands where the last one left off and the delay the buffer has learned carries
|
||||
* over. Only a long silence drops the timeline.
|
||||
* <p>A speaker has one timeline, {@code unwrap16(packetId) * 60} ticks, kept across talk
|
||||
* bursts so the delay the buffer has learned carries over. The official client keeps
|
||||
* numbering while its user is silent, so a burst usually starts well ahead of where the last
|
||||
* one stopped; once the talker has let go, playback skips to the next burst, holding the
|
||||
* same delay. Only a long silence drops the timeline.
|
||||
*
|
||||
* <p>An empty packet is the talker letting go of the key. It never enters the buffer; its
|
||||
* position is remembered, the burst ends once playback reaches it, and its slot is played as
|
||||
@@ -64,6 +64,14 @@ final class VoiceStream {
|
||||
private int lastSeq;
|
||||
private int[] stopMarks = new int[8];
|
||||
private int stopMarkCount;
|
||||
/** Where the audio last played from a packet ends; the talker resumes from there. */
|
||||
private int heard;
|
||||
private boolean heardAny;
|
||||
/** How far playback trailed the talker when they let go; the next burst is held back as much. */
|
||||
private int stopLag = FRAME_TICKS;
|
||||
/** Slots before this one are the silence ahead of a burst skipped to, not losses. */
|
||||
private int burstStart;
|
||||
private boolean skippedTo;
|
||||
/** Channels of the latest packet: TeamSpeak streams {@code OPUS_MUSIC} in stereo. */
|
||||
private int channels = 1;
|
||||
private boolean playing;
|
||||
@@ -76,7 +84,7 @@ final class VoiceStream {
|
||||
private int extraTicks;
|
||||
/** Frames of silence still to play before a fresh timeline starts, building up {@link #extraTicks}. */
|
||||
private int leadInFrames;
|
||||
/** Set when the timeline was dropped; the decoder follows on the next pull. */
|
||||
/** Set when a burst ended or the timeline was dropped: the talker's encoder starts afresh. */
|
||||
private boolean resetDecoder;
|
||||
/** Recently received sequence numbers, each at {@code seq % SEEN_WINDOW}. */
|
||||
private final int[] seen = new int[SEEN_WINDOW];
|
||||
@@ -150,6 +158,9 @@ final class VoiceStream {
|
||||
jitter.reset();
|
||||
epoch = EPOCH_UNSET;
|
||||
stopMarkCount = 0;
|
||||
heardAny = false;
|
||||
skippedTo = false;
|
||||
stopLag = FRAME_TICKS;
|
||||
resetDecoder = true;
|
||||
forgetSeen();
|
||||
}
|
||||
@@ -158,6 +169,7 @@ final class VoiceStream {
|
||||
int ticks = seq * FRAME_TICKS;
|
||||
if (end) {
|
||||
markStop(ticks);
|
||||
if (playing) stopLag = Math.max(jitter.margin(), ticks - jitter.position());
|
||||
return false;
|
||||
}
|
||||
boolean behind = !fresh && ticks - jitter.position() < 0;
|
||||
@@ -199,8 +211,6 @@ final class VoiceStream {
|
||||
boolean reset;
|
||||
synchronized (lock) {
|
||||
if (!playing) return IDLE;
|
||||
reset = resetDecoder;
|
||||
resetDecoder = false;
|
||||
packetChannels = channels;
|
||||
|
||||
if (leadInFrames > 0) {
|
||||
@@ -218,6 +228,7 @@ final class VoiceStream {
|
||||
}
|
||||
jitter.remainingSpan(remainingTicks());
|
||||
} else {
|
||||
skipToNextBurst();
|
||||
boolean empty = jitter.isEmpty();
|
||||
emptyPulls = empty ? emptyPulls + 1 : 0;
|
||||
if (empty && (stoppedAt(jitter.position()) || emptyPulls >= EMPTY_PULLS_TO_END)) {
|
||||
@@ -230,9 +241,12 @@ final class VoiceStream {
|
||||
source = Source.PACKET;
|
||||
packet = jitter.payload();
|
||||
played++;
|
||||
skippedTo = false;
|
||||
heard(jitter.position());
|
||||
}
|
||||
case MISSING -> {
|
||||
if (isStopMark(jitter.timestamp())) {
|
||||
if (isStopMark(jitter.timestamp())
|
||||
|| (skippedTo && jitter.timestamp() - burstStart < 0)) {
|
||||
source = Source.SILENCE;
|
||||
} else {
|
||||
missed++;
|
||||
@@ -240,6 +254,7 @@ final class VoiceStream {
|
||||
if (packet != null) {
|
||||
source = Source.RECOVER;
|
||||
recovered++;
|
||||
heard(jitter.position());
|
||||
} else {
|
||||
source = Source.CONCEAL;
|
||||
}
|
||||
@@ -254,6 +269,8 @@ final class VoiceStream {
|
||||
jitter.tick();
|
||||
}
|
||||
}
|
||||
reset = source != null && resetDecoder;
|
||||
if (reset) resetDecoder = false;
|
||||
}
|
||||
if (source == null) {
|
||||
markTalking(false);
|
||||
@@ -431,11 +448,36 @@ final class VoiceStream {
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Whether the talker let go at or shortly before {@code position}, so the burst is spoken out. */
|
||||
/**
|
||||
* Once the talker has let go, moves playback up to the next burst they started, which may
|
||||
* be numbered far ahead, trailing it by the delay held when they stopped. Call under the lock.
|
||||
*/
|
||||
private void skipToNextBurst() {
|
||||
int position = jitter.position();
|
||||
if (jitter.isEmpty() || !stoppedAt(position)) return;
|
||||
int next = jitter.earliest();
|
||||
int target = next - stopLag;
|
||||
if (target - position <= 0) return;
|
||||
jitter.seek(target);
|
||||
burstStart = next;
|
||||
skippedTo = true;
|
||||
resetDecoder = true;
|
||||
}
|
||||
|
||||
private void heard(int position) {
|
||||
heard = position;
|
||||
heardAny = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the talker let go between the last audio heard and {@code position}, so the
|
||||
* burst is spoken out. An earlier burst's stop does not count.
|
||||
*/
|
||||
private boolean stoppedAt(int position) {
|
||||
for (int i = 0; i < stopMarkCount; i++) {
|
||||
int behind = position - stopMarks[i];
|
||||
if (behind >= 0 && behind < STOP_MARK_WINDOW_TICKS) return true;
|
||||
if (behind < 0 || behind >= STOP_MARK_WINDOW_TICKS) continue;
|
||||
if (!heardAny || stopMarks[i] - heard >= 0) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -449,7 +491,14 @@ final class VoiceStream {
|
||||
}
|
||||
|
||||
private void stopLocked() {
|
||||
// Concealment past the last packet was a guess that the talker was still going. Take
|
||||
// it back, or the next burst, numbered on from that packet, lands behind playback.
|
||||
if (heardAny && jitter.position() - heard > 0) {
|
||||
if (!stoppedAt(jitter.position())) markStop(heard);
|
||||
jitter.seek(heard);
|
||||
}
|
||||
playing = false;
|
||||
resetDecoder = true;
|
||||
surplusFrames = 0;
|
||||
concealOwed = 0;
|
||||
leadInFrames = 0;
|
||||
|
||||
@@ -146,7 +146,7 @@ class VoiceStreamTest {
|
||||
assertTrue(offer(2, 2), "the next packet starts a new burst");
|
||||
assertEquals(0f, pull(), "the stop's slot is the talker's own silence, not a loss");
|
||||
assertEquals(0.02f, pull());
|
||||
assertEquals(0, opus.resets, "the decoder keeps its state between bursts");
|
||||
assertEquals(1, opus.resets, "each burst starts a fresh decoder, as the talker's encoder does");
|
||||
assertEquals(List.of(), opus.concealedFrameSizes);
|
||||
}
|
||||
|
||||
@@ -160,6 +160,71 @@ class VoiceStreamTest {
|
||||
assertEquals(List.of(true, false), talk);
|
||||
}
|
||||
|
||||
/** The official client keeps numbering through silence, so its next burst starts far ahead. */
|
||||
@Test
|
||||
void skipsToABurstNumberedAheadOfWhereTheLastStopped() {
|
||||
offer(0, 1);
|
||||
offer(1, 2);
|
||||
offerEnd(2);
|
||||
pull();
|
||||
pull();
|
||||
assertEquals(VoiceStream.IDLE, stream.pull(out, 1, 1.0));
|
||||
|
||||
assertTrue(offer(60, 3));
|
||||
List<Float> heard = new ArrayList<>();
|
||||
for (int i = 0; i < 8; i++) {
|
||||
offer(61 + i, 4 + i);
|
||||
heard.add(pull());
|
||||
}
|
||||
assertEquals(0.03f, heard.stream().filter(v -> v != 0f).findFirst().orElseThrow(),
|
||||
"the burst plays from its first packet: " + heard);
|
||||
assertTrue(heard.indexOf(0.03f) <= 2, "after at most the delay held before: " + heard);
|
||||
assertEquals(List.of(), opus.concealedFrameSizes);
|
||||
assertEquals(1, opus.resets);
|
||||
}
|
||||
|
||||
@Test
|
||||
void aBurstThatArrivesBeforeTheLastDrainedStartsAFreshDecoder() {
|
||||
offer(0, 1);
|
||||
offer(1, 2);
|
||||
offerEnd(2);
|
||||
pull();
|
||||
offer(60, 3);
|
||||
offer(61, 4);
|
||||
List<Float> heard = new ArrayList<>();
|
||||
for (int i = 0; i < 6; i++) heard.add(pull());
|
||||
assertTrue(heard.contains(0.02f) && heard.contains(0.03f), "both bursts play in full: " + heard);
|
||||
assertEquals(1, opus.resets);
|
||||
}
|
||||
|
||||
@Test
|
||||
void theBurstAfterALostStopPlaysFromItsFirstPacket() {
|
||||
offer(0, 1);
|
||||
offer(1, 2);
|
||||
while (stream.pull(out, 1, 1.0) != VoiceStream.IDLE) {
|
||||
// plays both, then conceals until it gives up on the stop
|
||||
}
|
||||
assertTrue(offer(2, 3), "the talker resumes where the last burst's audio ended");
|
||||
assertEquals(0.03f, pull());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aLateCopyOfTheLastPacketDoesNotHoldTheBurstOpen() {
|
||||
offer(0, 1);
|
||||
offer(1, 2);
|
||||
offerEnd(2);
|
||||
pull();
|
||||
pull();
|
||||
offer(1, 2); // a copy, after playback passed it
|
||||
assertEquals(VoiceStream.IDLE, stream.pull(out, 1, 1.0));
|
||||
|
||||
assertTrue(offer(3, 4));
|
||||
float sample = pull();
|
||||
// The late copy taught the buffer to wait a frame longer, so the start may be padded.
|
||||
for (int i = 0; i < 3 && sample <= 0; i++) sample = pull();
|
||||
assertEquals(0.04f, sample);
|
||||
}
|
||||
|
||||
@Test
|
||||
void playsALongPacketOneFrameAtATime() {
|
||||
stream.offer(0, CodecType.OPUS_VOICE, new byte[]{40, 0});
|
||||
|
||||
Reference in New Issue
Block a user