ericek111/ts3j
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Replace RMS voice detection with TeamSpeak's own RNN VAD

Browse Source 
Ports WebRTC's rnn_vad (as TS3 embeds it) to Java: LPC, pitch
estimation, spectral features and the RNN itself, feeding a
speech-probability detector that replaces the old SpeechDetector.
Also switches the volume-gate threshold from raw dBFS to
InputLevel's scale, matching TS3's own slider and range, with a
migration for settings saved under the old key.
This commit is contained in:
ericek111
2026-08-17 07:51:51 +00:00
parent 752676e863
commit e228dd4ad0
23 changed files with 2543 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/DesktopVoiceInput.java
View File

@@ -2,9 +2,12 @@ package com.ts3client.audio.desktop;
import com.github.manevolent.ts3j.enums.CodecType;
import com.ts3client.audio.AudioEnhancer;
import com.ts3client.audio.AudioFrameListener;
import com.ts3client.audio.InputLevel;
import com.ts3client.audio.OpusParameters;
import com.ts3client.audio.SpeechDetector;
import com.ts3client.audio.SpeechProbabilityDetector;
import com.ts3client.audio.VoiceInput;
import com.ts3client.audio.vad.RnnSpeechDetector;
import com.ts3client.config.Settings;
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -28,7 +31,23 @@ import java.util.function.Consumer;
*/
public final class DesktopVoiceInput implements VoiceInput {
private static final int HANGOVER_FRAMES = 15; // ~300 ms of tail after level drops
/**
* How long the gate stays open after the last active frame. TS3 counts 64 of its
* 10 ms preprocessor frames before closing, so the tail of a word is never clipped.
*/
private static final int HANGOVER_MS = 640;
/**
* Audio replayed when the gate opens, so a word's onset is not swallowed by the frame
* that detected it. TS3 keeps up to three 10 ms buffers ({@code vad_extrabuffersize}
* defaults to 2, plus one).
*/
private static final int PREROLL_MS = 30;
private static final int HANGOVER_FRAMES =
Math.max(1, HANGOVER_MS * AudioDevices.SAMPLE_RATE / 1000 / AudioDevices.FRAME_SIZE);
private static final int PREROLL_FRAMES =
Math.max(1, PREROLL_MS * AudioDevices.SAMPLE_RATE / 1000 / AudioDevices.FRAME_SIZE);
private final ConcurrentLinkedQueue<byte[]> queue = new ConcurrentLinkedQueue<>();
private final AtomicBoolean muted = new AtomicBoolean(false);
@@ -44,12 +63,14 @@ public final class DesktopVoiceInput implements VoiceInput {
private volatile double inputGain;
private volatile CodecType codec = CodecType.OPUS_VOICE;
private final SpeechDetector speechDetector = new SpeechDetector(AudioDevices.SAMPLE_RATE);
private final SpeechProbabilityDetector speechDetector =
new RnnSpeechDetector(AudioDevices.SAMPLE_RATE);
private final AudioEnhancer enhancer = new AudioEnhancer(AudioDevices.SAMPLE_RATE);
private volatile Consumer<Double> levelListener; // input level in dBFS
private volatile Consumer<Double> levelListener; // input level, InputLevel scale
private volatile Consumer<Boolean> talkListener; // local talk-state changes
private volatile Runnable mutedTalkListener; // speech detected while muted
private volatile AudioFrameListener monitorListener; // local monitoring of what is sent
private boolean mutedTalking;
private final String deviceName;
@@ -68,6 +89,11 @@ public final class DesktopVoiceInput implements VoiceInput {
private int hangover;
private boolean lastTransmitting;
// Ring of recent frames captured while the gate was shut.
private final float[][] preroll = new float[PREROLL_FRAMES][];
private int prerollTail;
private int prerollCount;
public DesktopVoiceInput(Settings settings) {
this.deviceName = settings.inputDevice;
this.mode = settings.inputMode;
@@ -195,6 +221,11 @@ public final class DesktopVoiceInput implements VoiceInput {
this.talkListener = l;
}
@Override
public void setMonitorListener(AudioFrameListener l) {
this.monitorListener = l;
}
@Override
public void setMutedTalkListener(Runnable l) {
this.mutedTalkListener = l;
@@ -305,34 +336,74 @@ public final class DesktopVoiceInput implements VoiceInput {
// stream bypasses them and is transmitted as captured.
if (!stereo) enhancer.process(mono, frameSamples);
double sumSq = 0;
for (int i = 0; i < frameSamples; i++) {
sumSq += (double) mono[i] * mono[i];
}
double rms = Math.sqrt(sumSq / frameSamples);
double db = (rms <= 1e-9) ? -100.0 : 20.0 * Math.log10(rms);
double db = InputLevel.toDb(mono, frameSamples);
Consumer<Double> ll = levelListener;
if (ll != null) ll.accept(db);
boolean wasOpen = transmitting.get();
boolean open = decideGate(db, mono);
setTransmitting(open);
if (open && !muted.get() && encoder != null) {
try {
packet = encoder.encode(stereo ? pcm : mono);
// On the opening edge, send the buffered lead-in first so the
// word's onset isn't lost to the frame that detected it.
if (!wasOpen) {
flushPreroll(stereo);
}
float[] sent = stereo ? pcm : mono;
packet = encoder.encode(sent);
monitor(sent, stereo ? encoderChannels : 1);
} catch (Exception ignored) {
}
}
if (!open) {
rememberForPreroll(stereo ? pcm : mono);
}
}
if (packet != null && packet.length > 0) {
queue.offer(packet);
// Guard against unbounded growth if the network stalls.
while (queue.size() > 10) queue.poll();
offer(packet);
}
}
}
private void offer(byte[] packet) {
queue.offer(packet);
// Guard against unbounded growth if the network stalls.
while (queue.size() > 10) queue.poll();
}
/** Keeps the most recent frames while the gate is shut, for {@link #flushPreroll}. */
private void rememberForPreroll(float[] frame) {
float[] slot = preroll[prerollTail];
if (slot == null || slot.length != frame.length) {
slot = new float[frame.length];
preroll[prerollTail] = slot;
}
System.arraycopy(frame, 0, slot, 0, frame.length);
prerollTail = (prerollTail + 1) % PREROLL_FRAMES;
if (prerollCount < PREROLL_FRAMES) prerollCount++;
}
/** Encodes and queues the buffered lead-in, oldest first. Call under {@link #encoderLock}. */
private void flushPreroll(boolean stereo) {
int expected = stereo ? AudioDevices.FRAME_SIZE * encoderChannels : AudioDevices.FRAME_SIZE;
for (int i = 0; i < prerollCount; i++) {
int idx = (prerollTail - prerollCount + i + PREROLL_FRAMES) % PREROLL_FRAMES;
float[] frame = preroll[idx];
// A channel-count change between capture and flush invalidates the buffer.
if (frame == null || frame.length != expected) continue;
try {
byte[] p = encoder.encode(frame);
if (p != null && p.length > 0) offer(p);
monitor(frame, stereo ? encoderChannels : 1);
} catch (Exception ignored) {
}
}
prerollCount = 0;
}
private boolean decideGate(double db, float[] pcm) {
if (muted.get()) {
hangover = 0;
@@ -353,8 +424,10 @@ public final class DesktopVoiceInput implements VoiceInput {
}
/**
* Applies the selected VAD mode (volume gate, speech probability, or both) with
* a deactivation-delay hangover so trailing syllables aren't clipped.
* Applies the selected VAD mode with a hangover so trailing syllables aren't clipped.
*
* <p>The modes match the TS3 client's: the volume gate alone, the speech detector
* alone, or both together. Volume Gate skips the detector entirely, as TS3 does.
*/
private boolean voiceActivated(double db, float[] pcm) {
boolean detected;
@@ -363,11 +436,11 @@ public final class DesktopVoiceInput implements VoiceInput {
detected = db >= thresholdDb;
break;
case AUTOMATIC:
detected = speechDetector.process(pcm) >= speechThreshold;
detected = speechDetector.process(pcm, AudioDevices.FRAME_SIZE) >= speechThreshold;
break;
case HYBRID:
default:
double probability = speechDetector.process(pcm);
double probability = speechDetector.process(pcm, AudioDevices.FRAME_SIZE);
detected = db >= thresholdDb && probability >= speechThreshold;
break;
}
@@ -395,6 +468,12 @@ public final class DesktopVoiceInput implements VoiceInput {
mutedTalking = talking;
}
/** Hands a transmitted frame to the monitor, if one is attached. */
private void monitor(float[] frame, int channels) {
AudioFrameListener l = monitorListener;
if (l != null) l.onFrame(frame, channels);
}
private void setTransmitting(boolean t) {
transmitting.set(t);
if (t != lastTransmitting) {