diff --git a/ts3-client/core/src/main/java/eu/lixko/ts3j/client/audio/JitterBuffer.java b/ts3-client/core/src/main/java/eu/lixko/ts3j/client/audio/JitterBuffer.java index de577e0..606e843 100644 --- a/ts3-client/core/src/main/java/eu/lixko/ts3j/client/audio/JitterBuffer.java +++ b/ts3-client/core/src/main/java/eu/lixko/ts3j/client/audio/JitterBuffer.java @@ -207,14 +207,54 @@ public final class JitterBuffer { return total; } - /** True when no packet is held and playback has nothing left to drain. */ + /** + * True when playback has nothing left to drain. A packet that arrived after playback + * passed it may still be held, for the timing it reported, but it can never be played. + */ public boolean isEmpty() { + if (resetState) { + for (int i = 0; i < MAX_PACKETS; i++) { + if (data[i] != null) return false; + } + return true; + } for (int i = 0; i < MAX_PACKETS; i++) { - if (data[i] != null) return false; + if (data[i] != null && gt(timestamp[i] + span[i], pointerTimestamp)) return false; } return true; } + /** Timestamp of the earliest packet still to be played; only meaningful when not {@link #isEmpty}. */ + public int earliest() { + boolean found = false; + int earliest = pointerTimestamp; + for (int i = 0; i < MAX_PACKETS; i++) { + if (data[i] == null || (!resetState && !gt(timestamp[i] + span[i], pointerTimestamp))) continue; + if (!found || lt(timestamp[i], earliest)) { + earliest = timestamp[i]; + found = true; + } + } + return earliest; + } + + /** + * Moves playback to {@code timestamp}, keeping the delay estimate: held packets measure + * their slack against the moved timeline. Does nothing before the first packet has + * anchored the timeline. + */ + public void seek(int timestamp) { + if (resetState) return; + int delta = timestamp - pointerTimestamp; + for (int i = 0; i < MAX_PACKETS; i++) { + if (data[i] != null && arrival[i] != 0) arrival[i] += delta; + } + pointerTimestamp = timestamp; + nextStop = timestamp; + buffered = 0; + lostCount = 0; + } + /** * Drops every held packet and forgets both the timeline and the network estimate. * diff --git a/ts3-client/core/src/main/java/eu/lixko/ts3j/client/audio/VoiceStream.java b/ts3-client/core/src/main/java/eu/lixko/ts3j/client/audio/VoiceStream.java index 2fcfb3f..dfea8b8 100644 --- a/ts3-client/core/src/main/java/eu/lixko/ts3j/client/audio/VoiceStream.java +++ b/ts3-client/core/src/main/java/eu/lixko/ts3j/client/audio/VoiceStream.java @@ -10,11 +10,11 @@ import java.util.function.BiConsumer; * One speaker's incoming voice: packets go into an adaptive {@link JitterBuffer} as they * arrive, and every {@link #pull} plays out the next 20 ms, concealing what went missing. * - *

The buffer is driven the way the official client drives it. A speaker has one timeline, - * {@code unwrap16(packetId) * 60} ticks, which is never re-anchored between talk bursts: the - * sender stops numbering when it stops talking and playback stops at the same point, so the - * next burst lands where the last one left off and the delay the buffer has learned carries - * over. Only a long silence drops the timeline. + *

A speaker has one timeline, {@code unwrap16(packetId) * 60} ticks, kept across talk + * bursts so the delay the buffer has learned carries over. The official client keeps + * numbering while its user is silent, so a burst usually starts well ahead of where the last + * one stopped; once the talker has let go, playback skips to the next burst, holding the + * same delay. Only a long silence drops the timeline. * *

An empty packet is the talker letting go of the key. It never enters the buffer; its * position is remembered, the burst ends once playback reaches it, and its slot is played as @@ -64,6 +64,14 @@ final class VoiceStream { private int lastSeq; private int[] stopMarks = new int[8]; private int stopMarkCount; + /** Where the audio last played from a packet ends; the talker resumes from there. */ + private int heard; + private boolean heardAny; + /** How far playback trailed the talker when they let go; the next burst is held back as much. */ + private int stopLag = FRAME_TICKS; + /** Slots before this one are the silence ahead of a burst skipped to, not losses. */ + private int burstStart; + private boolean skippedTo; /** Channels of the latest packet: TeamSpeak streams {@code OPUS_MUSIC} in stereo. */ private int channels = 1; private boolean playing; @@ -76,7 +84,7 @@ final class VoiceStream { private int extraTicks; /** Frames of silence still to play before a fresh timeline starts, building up {@link #extraTicks}. */ private int leadInFrames; - /** Set when the timeline was dropped; the decoder follows on the next pull. */ + /** Set when a burst ended or the timeline was dropped: the talker's encoder starts afresh. */ private boolean resetDecoder; /** Recently received sequence numbers, each at {@code seq % SEEN_WINDOW}. */ private final int[] seen = new int[SEEN_WINDOW]; @@ -150,6 +158,9 @@ final class VoiceStream { jitter.reset(); epoch = EPOCH_UNSET; stopMarkCount = 0; + heardAny = false; + skippedTo = false; + stopLag = FRAME_TICKS; resetDecoder = true; forgetSeen(); } @@ -158,6 +169,7 @@ final class VoiceStream { int ticks = seq * FRAME_TICKS; if (end) { markStop(ticks); + if (playing) stopLag = Math.max(jitter.margin(), ticks - jitter.position()); return false; } boolean behind = !fresh && ticks - jitter.position() < 0; @@ -199,8 +211,6 @@ final class VoiceStream { boolean reset; synchronized (lock) { if (!playing) return IDLE; - reset = resetDecoder; - resetDecoder = false; packetChannels = channels; if (leadInFrames > 0) { @@ -218,6 +228,7 @@ final class VoiceStream { } jitter.remainingSpan(remainingTicks()); } else { + skipToNextBurst(); boolean empty = jitter.isEmpty(); emptyPulls = empty ? emptyPulls + 1 : 0; if (empty && (stoppedAt(jitter.position()) || emptyPulls >= EMPTY_PULLS_TO_END)) { @@ -230,9 +241,12 @@ final class VoiceStream { source = Source.PACKET; packet = jitter.payload(); played++; + skippedTo = false; + heard(jitter.position()); } case MISSING -> { - if (isStopMark(jitter.timestamp())) { + if (isStopMark(jitter.timestamp()) + || (skippedTo && jitter.timestamp() - burstStart < 0)) { source = Source.SILENCE; } else { missed++; @@ -240,6 +254,7 @@ final class VoiceStream { if (packet != null) { source = Source.RECOVER; recovered++; + heard(jitter.position()); } else { source = Source.CONCEAL; } @@ -254,6 +269,8 @@ final class VoiceStream { jitter.tick(); } } + reset = source != null && resetDecoder; + if (reset) resetDecoder = false; } if (source == null) { markTalking(false); @@ -431,11 +448,36 @@ final class VoiceStream { return false; } - /** Whether the talker let go at or shortly before {@code position}, so the burst is spoken out. */ + /** + * Once the talker has let go, moves playback up to the next burst they started, which may + * be numbered far ahead, trailing it by the delay held when they stopped. Call under the lock. + */ + private void skipToNextBurst() { + int position = jitter.position(); + if (jitter.isEmpty() || !stoppedAt(position)) return; + int next = jitter.earliest(); + int target = next - stopLag; + if (target - position <= 0) return; + jitter.seek(target); + burstStart = next; + skippedTo = true; + resetDecoder = true; + } + + private void heard(int position) { + heard = position; + heardAny = true; + } + + /** + * Whether the talker let go between the last audio heard and {@code position}, so the + * burst is spoken out. An earlier burst's stop does not count. + */ private boolean stoppedAt(int position) { for (int i = 0; i < stopMarkCount; i++) { int behind = position - stopMarks[i]; - if (behind >= 0 && behind < STOP_MARK_WINDOW_TICKS) return true; + if (behind < 0 || behind >= STOP_MARK_WINDOW_TICKS) continue; + if (!heardAny || stopMarks[i] - heard >= 0) return true; } return false; } @@ -449,7 +491,14 @@ final class VoiceStream { } private void stopLocked() { + // Concealment past the last packet was a guess that the talker was still going. Take + // it back, or the next burst, numbered on from that packet, lands behind playback. + if (heardAny && jitter.position() - heard > 0) { + if (!stoppedAt(jitter.position())) markStop(heard); + jitter.seek(heard); + } playing = false; + resetDecoder = true; surplusFrames = 0; concealOwed = 0; leadInFrames = 0; diff --git a/ts3-client/core/src/test/java/eu/lixko/ts3j/client/audio/VoiceStreamTest.java b/ts3-client/core/src/test/java/eu/lixko/ts3j/client/audio/VoiceStreamTest.java index 7b0c6f0..8720d7f 100644 --- a/ts3-client/core/src/test/java/eu/lixko/ts3j/client/audio/VoiceStreamTest.java +++ b/ts3-client/core/src/test/java/eu/lixko/ts3j/client/audio/VoiceStreamTest.java @@ -146,7 +146,7 @@ class VoiceStreamTest { assertTrue(offer(2, 2), "the next packet starts a new burst"); assertEquals(0f, pull(), "the stop's slot is the talker's own silence, not a loss"); assertEquals(0.02f, pull()); - assertEquals(0, opus.resets, "the decoder keeps its state between bursts"); + assertEquals(1, opus.resets, "each burst starts a fresh decoder, as the talker's encoder does"); assertEquals(List.of(), opus.concealedFrameSizes); } @@ -160,6 +160,71 @@ class VoiceStreamTest { assertEquals(List.of(true, false), talk); } + /** The official client keeps numbering through silence, so its next burst starts far ahead. */ + @Test + void skipsToABurstNumberedAheadOfWhereTheLastStopped() { + offer(0, 1); + offer(1, 2); + offerEnd(2); + pull(); + pull(); + assertEquals(VoiceStream.IDLE, stream.pull(out, 1, 1.0)); + + assertTrue(offer(60, 3)); + List heard = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + offer(61 + i, 4 + i); + heard.add(pull()); + } + assertEquals(0.03f, heard.stream().filter(v -> v != 0f).findFirst().orElseThrow(), + "the burst plays from its first packet: " + heard); + assertTrue(heard.indexOf(0.03f) <= 2, "after at most the delay held before: " + heard); + assertEquals(List.of(), opus.concealedFrameSizes); + assertEquals(1, opus.resets); + } + + @Test + void aBurstThatArrivesBeforeTheLastDrainedStartsAFreshDecoder() { + offer(0, 1); + offer(1, 2); + offerEnd(2); + pull(); + offer(60, 3); + offer(61, 4); + List heard = new ArrayList<>(); + for (int i = 0; i < 6; i++) heard.add(pull()); + assertTrue(heard.contains(0.02f) && heard.contains(0.03f), "both bursts play in full: " + heard); + assertEquals(1, opus.resets); + } + + @Test + void theBurstAfterALostStopPlaysFromItsFirstPacket() { + offer(0, 1); + offer(1, 2); + while (stream.pull(out, 1, 1.0) != VoiceStream.IDLE) { + // plays both, then conceals until it gives up on the stop + } + assertTrue(offer(2, 3), "the talker resumes where the last burst's audio ended"); + assertEquals(0.03f, pull()); + } + + @Test + void aLateCopyOfTheLastPacketDoesNotHoldTheBurstOpen() { + offer(0, 1); + offer(1, 2); + offerEnd(2); + pull(); + pull(); + offer(1, 2); // a copy, after playback passed it + assertEquals(VoiceStream.IDLE, stream.pull(out, 1, 1.0)); + + assertTrue(offer(3, 4)); + float sample = pull(); + // The late copy taught the buffer to wait a frame longer, so the start may be padded. + for (int i = 0; i < 3 && sample <= 0; i++) sample = pull(); + assertEquals(0.04f, sample); + } + @Test void playsALongPacketOneFrameAtATime() { stream.offer(0, CodecType.OPUS_VOICE, new byte[]{40, 0});