Add extra receive buffering and decode Opus FEC on loss
A new Extra buffering setting (Network page) holds more playback delay than the jitter buffer measures it needs, so late packets still make it in time on a bad connection. A lost frame is now rebuilt from the FEC copy in the next packet when that one is already held, and the Connection Info shows how many were recovered. Held packets' arrival marks now move with the timeline when the delay changes; libspeex leaves them, which made a multi-frame jump count twice. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -184,8 +184,13 @@ public final class JitterBuffer {
|
||||
this.subWindowSize = this.windowSize / SUB_WINDOWS;
|
||||
}
|
||||
|
||||
/** Sets the extra delay held on top of the measured optimum. */
|
||||
/**
|
||||
* Sets the extra delay held on top of the measured optimum. The timings already recorded
|
||||
* are re-based onto it, so the delay moves at the next {@link #tick} rather than once the
|
||||
* estimate has turned over.
|
||||
*/
|
||||
public void setMargin(int margin) {
|
||||
shiftTimings(bufferMargin - margin);
|
||||
this.bufferMargin = margin;
|
||||
}
|
||||
|
||||
@@ -478,6 +483,12 @@ public final class JitterBuffer {
|
||||
shiftTimings(-optimal);
|
||||
pointerTimestamp += optimal;
|
||||
if (optimal < 0) interpolationRequested = -optimal;
|
||||
// Held packets measure their slack against when they arrived, so move that with the
|
||||
// timeline too. libspeex does not, which makes a delay jump of several frames count
|
||||
// twice: every packet already waiting reports the old shortfall again on playout.
|
||||
for (int i = 0; i < MAX_PACKETS; i++) {
|
||||
if (data[i] != null && arrival[i] != 0) arrival[i] += optimal;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -47,6 +47,8 @@ public final class StreamingVoiceOutput implements VoiceOutput {
|
||||
|
||||
private volatile String outputDevice;
|
||||
private volatile double masterVolume = 1.0;
|
||||
/** Extra playback delay, in jitter buffer ticks. */
|
||||
private volatile int extraDelayTicks;
|
||||
private volatile boolean deafened = false;
|
||||
|
||||
/** Notified (clientId, talking) on an audio thread when a speaker starts/stops. */
|
||||
@@ -149,6 +151,13 @@ public final class StreamingVoiceOutput implements VoiceOutput {
|
||||
this.outputDevice = device == null ? "" : device;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setReceiveBuffer(int ms) {
|
||||
int frames = Math.max(0, ms) * (VoiceFormat.SAMPLE_RATE / 1000) / VoiceFormat.FRAME_SIZE;
|
||||
extraDelayTicks = frames * JitterBuffer.TEAMSPEAK_TICKS_PER_FRAME;
|
||||
for (VoiceStream s : streams.values()) s.setExtraDelay(extraDelayTicks);
|
||||
}
|
||||
|
||||
/** Entry point wired into {@code client.setVoiceHandler(...)}. */
|
||||
@Override
|
||||
public void handleVoice(PacketBody0Voice voice) {
|
||||
@@ -168,6 +177,7 @@ public final class StreamingVoiceOutput implements VoiceOutput {
|
||||
VoiceStream stream = streams.computeIfAbsent(clientId, id -> {
|
||||
VoiceStream created = new VoiceStream(id, opus, this::fireTalk);
|
||||
created.setTap(voiceTap);
|
||||
created.setExtraDelay(extraDelayTicks);
|
||||
return created;
|
||||
});
|
||||
if (stream.offer(packetId, codec, data)) playout.started(stream);
|
||||
|
||||
@@ -34,6 +34,12 @@ public interface VoiceOutput {
|
||||
|
||||
void setOutputDevice(String device);
|
||||
|
||||
/**
|
||||
* Delays every speaker by {@code ms} more than the network calls for, trading latency for
|
||||
* fewer dropouts on a bad connection.
|
||||
*/
|
||||
void setReceiveBuffer(int ms);
|
||||
|
||||
void removeClient(int clientId);
|
||||
|
||||
void shutdown();
|
||||
|
||||
@@ -6,23 +6,25 @@ package com.ts3client.audio;
|
||||
*
|
||||
* @param received voice packets that arrived, copies included
|
||||
* @param played frames played from a packet
|
||||
* @param missed frames concealed because their packet was not there when due
|
||||
* @param missed frames whose packet was not there when due
|
||||
* @param recovered of those, frames rebuilt from the error correction copy in the next packet
|
||||
* rather than concealed
|
||||
* @param outOfOrder packets that arrived after a later one
|
||||
* @param late packets that arrived after their frame was concealed, so were dropped;
|
||||
* each one is also among {@code missed}
|
||||
* @param duplicates copies of a packet already received, discarded
|
||||
*/
|
||||
public record VoiceReceiveStats(long received, long played, long missed, long outOfOrder, long late,
|
||||
public record VoiceReceiveStats(long received, long played, long missed, long recovered, long outOfOrder, long late,
|
||||
long duplicates) {
|
||||
|
||||
public static final VoiceReceiveStats NONE = new VoiceReceiveStats(0, 0, 0, 0, 0, 0);
|
||||
public static final VoiceReceiveStats NONE = new VoiceReceiveStats(0, 0, 0, 0, 0, 0, 0);
|
||||
|
||||
public VoiceReceiveStats plus(VoiceReceiveStats o) {
|
||||
return new VoiceReceiveStats(received + o.received, played + o.played, missed + o.missed,
|
||||
outOfOrder + o.outOfOrder, late + o.late, duplicates + o.duplicates);
|
||||
recovered + o.recovered, outOfOrder + o.outOfOrder, late + o.late, duplicates + o.duplicates);
|
||||
}
|
||||
|
||||
/** Frames that were due, whether played or concealed. */
|
||||
/** Frames that were due, whether played or missed. */
|
||||
public long due() {
|
||||
return played + missed;
|
||||
}
|
||||
|
||||
@@ -20,6 +20,11 @@ import java.util.function.BiConsumer;
|
||||
* position is remembered, the burst ends once playback reaches it, and its slot is played as
|
||||
* silence rather than concealed, since nothing was lost there.
|
||||
*
|
||||
* <p>A frame that did go missing is rebuilt from the error correction copy in the packet after
|
||||
* it, when that one is already held; only otherwise is it concealed. An
|
||||
* {@linkplain #setExtraDelay extra delay} makes that, and late packets, more likely to be in
|
||||
* time.
|
||||
*
|
||||
* <p>The stream keeps no clock of its own: whoever pulls sets the pace, one call per
|
||||
* 20 ms frame. {@link #offer} and {@link #stop} may be called from any thread; pulls
|
||||
* must come from one thread at a time.
|
||||
@@ -46,7 +51,7 @@ final class VoiceStream {
|
||||
private static final int SEEN_WINDOW = 1024;
|
||||
|
||||
/** What a pull plays, decided under the lock and produced outside it. */
|
||||
private enum Source { PACKET, CONCEAL, SILENCE, SURPLUS }
|
||||
private enum Source { PACKET, RECOVER, CONCEAL, SILENCE, SURPLUS }
|
||||
|
||||
final int clientId;
|
||||
private final OpusCodec opus;
|
||||
@@ -67,6 +72,10 @@ final class VoiceStream {
|
||||
private int surplusFrames;
|
||||
/** Frames of concealment still owed to a delay increase. */
|
||||
private int concealOwed;
|
||||
/** Ticks of delay held on top of what the jitter buffer measures it needs. */
|
||||
private int extraTicks;
|
||||
/** Frames of silence still to play before a fresh timeline starts, building up {@link #extraTicks}. */
|
||||
private int leadInFrames;
|
||||
/** Set when the timeline was dropped; the decoder follows on the next pull. */
|
||||
private boolean resetDecoder;
|
||||
/** Recently received sequence numbers, each at {@code seq % SEEN_WINDOW}. */
|
||||
@@ -75,6 +84,7 @@ final class VoiceStream {
|
||||
private long received;
|
||||
private long played;
|
||||
private long missed;
|
||||
private long recovered;
|
||||
private long outOfOrder;
|
||||
private long late;
|
||||
private long duplicates;
|
||||
@@ -107,6 +117,17 @@ final class VoiceStream {
|
||||
this.tap = tap;
|
||||
}
|
||||
|
||||
/**
|
||||
* Holds {@code ticks} more delay than the network calls for, for a better chance that
|
||||
* late packets and those carrying a lost one's error correction copy arrive in time.
|
||||
*/
|
||||
void setExtraDelay(int ticks) {
|
||||
synchronized (lock) {
|
||||
extraTicks = Math.max(0, ticks);
|
||||
jitter.setMargin(FRAME_TICKS + extraTicks);
|
||||
}
|
||||
}
|
||||
|
||||
private static int channelsFor(CodecType codec) {
|
||||
return codec == CodecType.OPUS_MUSIC ? 2 : 1;
|
||||
}
|
||||
@@ -154,6 +175,8 @@ final class VoiceStream {
|
||||
playing = true;
|
||||
started = true;
|
||||
emptyPulls = 0;
|
||||
// The buffer would otherwise reach the extra delay by concealing frames mid-speech.
|
||||
if (fresh) leadInFrames = extraTicks / FRAME_TICKS;
|
||||
}
|
||||
}
|
||||
markTalking(true);
|
||||
@@ -180,7 +203,10 @@ final class VoiceStream {
|
||||
resetDecoder = false;
|
||||
packetChannels = channels;
|
||||
|
||||
if (surplusFrames > 0 || concealOwed > 0) {
|
||||
if (leadInFrames > 0) {
|
||||
leadInFrames--;
|
||||
source = Source.SILENCE;
|
||||
} else if (surplusFrames > 0 || concealOwed > 0) {
|
||||
// Still playing out what the last get() handed over.
|
||||
if (surplusFrames > 0) {
|
||||
source = Source.SURPLUS;
|
||||
@@ -209,8 +235,14 @@ final class VoiceStream {
|
||||
if (isStopMark(jitter.timestamp())) {
|
||||
source = Source.SILENCE;
|
||||
} else {
|
||||
source = Source.CONCEAL;
|
||||
missed++;
|
||||
packet = jitter.nextPayload();
|
||||
if (packet != null) {
|
||||
source = Source.RECOVER;
|
||||
recovered++;
|
||||
} else {
|
||||
source = Source.CONCEAL;
|
||||
}
|
||||
}
|
||||
}
|
||||
default -> {
|
||||
@@ -258,6 +290,11 @@ final class VoiceStream {
|
||||
play(pcm, 0, frames, decoderChannels, out, outChannels, gain);
|
||||
return frames;
|
||||
}
|
||||
case RECOVER -> {
|
||||
int frames = decoderFor(decoderChannels == 0 ? packetChannels : decoderChannels).recover(packet, pcm, FRAME);
|
||||
play(pcm, 0, frames, decoderChannels, out, outChannels, gain);
|
||||
return frames;
|
||||
}
|
||||
default -> {
|
||||
int frames = decoderFor(packetChannels).decode(packet, pcm);
|
||||
int played = Math.min(frames, FRAME);
|
||||
@@ -372,7 +409,7 @@ final class VoiceStream {
|
||||
/** How this speaker's voice has been coming in since the stream was created. */
|
||||
VoiceReceiveStats stats() {
|
||||
synchronized (lock) {
|
||||
return new VoiceReceiveStats(received, played, missed, outOfOrder, late, duplicates);
|
||||
return new VoiceReceiveStats(received, played, missed, recovered, outOfOrder, late, duplicates);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -415,6 +452,7 @@ final class VoiceStream {
|
||||
playing = false;
|
||||
surplusFrames = 0;
|
||||
concealOwed = 0;
|
||||
leadInFrames = 0;
|
||||
emptyPulls = 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,14 @@ public interface OpusDecoder extends AutoCloseable {
|
||||
*/
|
||||
int conceal(float[] out, int frameSize);
|
||||
|
||||
/**
|
||||
* Rebuilds {@code frameSize} samples per channel for a lost packet from the forward error
|
||||
* correction copy carried in {@code next}, the packet that followed it. Falls back to
|
||||
* concealment by itself when {@code next} carries none. {@code next} is then decoded as
|
||||
* usual.
|
||||
*/
|
||||
int recover(byte[] next, float[] out, int frameSize);
|
||||
|
||||
/** Forgets the stream's state, as at the start of a new talk burst. */
|
||||
void reset();
|
||||
|
||||
|
||||
@@ -133,6 +133,13 @@ public final class Settings {
|
||||
/** How many times every voice packet is sent again. */
|
||||
public int redundancyCopies = 1;
|
||||
public static final int MAX_REDUNDANCY_COPIES = 4;
|
||||
/**
|
||||
* Milliseconds of playback delay held on top of what the jitter buffer measures it needs,
|
||||
* so late packets and error correction copies still make it on a bad connection.
|
||||
*/
|
||||
public int receiveBuffer = 0;
|
||||
/** The {@link #receiveBuffer} choices the settings offer; whole 20 ms voice frames. */
|
||||
public static final int[] RECEIVE_BUFFERS = {0, 20, 40, 60, 80, 120, 200, 300, 500};
|
||||
/** Master playback gain, 0..1 (may exceed 1 for boost up to 2). */
|
||||
public double outputVolume = 1.0;
|
||||
/**
|
||||
@@ -291,6 +298,7 @@ public final class Settings {
|
||||
music = parseB(props.getProperty("music"), music);
|
||||
redundancyDelay = parseI(props.getProperty("redundancyDelay"), redundancyDelay);
|
||||
redundancyCopies = parseI(props.getProperty("redundancyCopies"), redundancyCopies);
|
||||
receiveBuffer = parseI(props.getProperty("receiveBuffer"), receiveBuffer);
|
||||
outputVolume = parseD(props.getProperty("outputVolume"), outputVolume);
|
||||
masterVolume = parseD(props.getProperty("masterVolume"), masterVolume);
|
||||
inputVolume = parseD(props.getProperty("inputVolume"), inputVolume);
|
||||
@@ -351,6 +359,7 @@ public final class Settings {
|
||||
props.setProperty("music", Boolean.toString(music));
|
||||
props.setProperty("redundancyDelay", Integer.toString(redundancyDelay));
|
||||
props.setProperty("redundancyCopies", Integer.toString(redundancyCopies));
|
||||
props.setProperty("receiveBuffer", Integer.toString(receiveBuffer));
|
||||
props.setProperty("outputVolume", Double.toString(outputVolume));
|
||||
props.setProperty("masterVolume", Double.toString(masterVolume));
|
||||
props.setProperty("inputVolume", Double.toString(inputVolume));
|
||||
|
||||
@@ -121,6 +121,7 @@ public final class ConnectionStatsFormat {
|
||||
new Row("Received", known ? count(s.received()) : UNKNOWN),
|
||||
new Row("Played", known ? countOf(s.played(), s.due()) : UNKNOWN),
|
||||
new Row("Missed", known ? countOf(s.missed(), s.due()) : UNKNOWN),
|
||||
new Row("Recovered (FEC)", known ? countOf(s.recovered(), s.missed()) : UNKNOWN),
|
||||
new Row("Out of order", known ? countOf(s.outOfOrder(), s.received()) : UNKNOWN),
|
||||
new Row("Dropped (too late)", known ? countOf(s.late(), s.received()) : UNKNOWN),
|
||||
new Row("Discarded (copies)", known ? countOf(s.duplicates(), s.received()) : UNKNOWN));
|
||||
|
||||
@@ -285,6 +285,7 @@ public final class TeamspeakConnection implements TS3Listener {
|
||||
model.clear();
|
||||
playback = audio.createOutput(settings);
|
||||
playback.setMasterVolume(settings.effectiveOutputVolume());
|
||||
playback.setReceiveBuffer(settings.receiveBuffer);
|
||||
playback.setTalkListener((clientId, talking) -> {
|
||||
ClientEntry c = model.getClient(clientId);
|
||||
if (c != null) c.talking = talking;
|
||||
|
||||
Reference in New Issue
Block a user