ericek111/ts3j
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Replace RMS voice detection with TeamSpeak's own RNN VAD

Browse Source 
Ports WebRTC's rnn_vad (as TS3 embeds it) to Java: LPC, pitch
estimation, spectral features and the RNN itself, feeding a
speech-probability detector that replaces the old SpeechDetector.
Also switches the volume-gate threshold from raw dBFS to
InputLevel's scale, matching TS3's own slider and range, with a
migration for settings saved under the old key.
This commit is contained in:
ericek111
2026-08-17 07:51:51 +00:00
parent 752676e863
commit e228dd4ad0
23 changed files with 2543 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

257
ts3-client/swing/src/main/java/com/ts3client/ui/SettingsDialog.java
View File

@@ -1,9 +1,9 @@
package com.ts3client.ui;
import com.ts3client.audio.InputLevel;
import com.ts3client.audio.OpusParameters;
import com.ts3client.audio.VoiceInput;
import com.ts3client.audio.VoiceOutput;
import com.ts3client.audio.desktop.AudioCapture;
import com.ts3client.audio.desktop.AudioDevices;
import com.ts3client.config.Settings;
import com.ts3client.hotkey.Hotkey;
@@ -23,6 +23,7 @@ import javax.swing.JPanel;
import javax.swing.JRadioButton;
import javax.swing.JSlider;
import javax.swing.JTabbedPane;
import javax.swing.JToggleButton;
import javax.swing.Scrollable;
import javax.swing.SwingUtilities;
import java.awt.BorderLayout;
@@ -80,6 +81,9 @@ public final class SettingsDialog extends JDialog {
private JLabel thresholdLabel;
private JLabel speechLabel;
private LevelMeter meter;
private JToggleButton testButton;
private JCheckBox loopbackCheck;
private JLabel talkIndicator;
private JButton pttKeyButton;
private JSlider bitrateSlider;
private JLabel bitrateLabel;
@@ -88,8 +92,7 @@ public final class SettingsDialog extends JDialog {
private JCheckBox fecCheck;
private JCheckBox musicCheck;
private volatile boolean meterRunning;
private Thread meterThread;
private final MicrophoneTest micTest = new MicrophoneTest(this::onTestLevel, this::onTestTalking);
public SettingsDialog(Frame owner, Settings settings,
VoiceInput liveMic, VoiceOutput livePlayback,
@@ -136,7 +139,7 @@ public final class SettingsDialog extends JDialog {
addWindowListener(new java.awt.event.WindowAdapter() {
@Override
public void windowClosed(java.awt.event.WindowEvent e) {
stopMeter();
micTest.stop();
}
});
@@ -144,7 +147,6 @@ public final class SettingsDialog extends JDialog {
setSize(new Dimension(480, 540));
setMinimumSize(new Dimension(420, 360));
setLocationRelativeTo(owner);
startMeter();
}
private JPanel buildDevicesTab() {
@@ -181,10 +183,10 @@ public final class SettingsDialog extends JDialog {
outputVol.addChangeListener(e -> {
if (livePlayback != null) livePlayback.setMasterVolume(outputVol.getValue() / 100.0);
});
inputGain.addChangeListener(e -> {
if (liveMic != null) liveMic.setInputGain(inputGain.getValue() / 100.0);
});
inputCombo.addActionListener(e -> restartMeter());
inputGain.addChangeListener(e ->
applyLive(m -> m.setInputGain(inputGain.getValue() / 100.0)));
inputCombo.addActionListener(e -> restartTest());
outputCombo.addActionListener(e -> micTest.setOutputDevice(comboValue(outputCombo)));
c.gridx = 0;
c.gridy = row++;
@@ -223,19 +225,18 @@ public final class SettingsDialog extends JDialog {
Runnable syncNoise = () -> {
denoiseLevel.setEnabled(denoiseCheck.isSelected());
if (liveMic != null) {
liveMic.setNoiseSuppression(denoiseCheck.isSelected());
liveMic.setDenoiserLevel(denoiseLevel.getValue() / 100.0);
liveMic.setTypingAttenuation(typingCheck.isSelected());
liveMic.setAgc(agcCheck.isSelected());
}
applyLive(m -> {
m.setNoiseSuppression(denoiseCheck.isSelected());
m.setDenoiserLevel(denoiseLevel.getValue() / 100.0);
m.setTypingAttenuation(typingCheck.isSelected());
m.setAgc(agcCheck.isSelected());
});
};
denoiseCheck.addActionListener(e -> syncNoise.run());
typingCheck.addActionListener(e -> syncNoise.run());
agcCheck.addActionListener(e -> syncNoise.run());
denoiseLevel.addChangeListener(e -> {
if (liveMic != null) liveMic.setDenoiserLevel(denoiseLevel.getValue() / 100.0);
});
denoiseLevel.addChangeListener(e ->
applyLive(m -> m.setDenoiserLevel(denoiseLevel.getValue() / 100.0)));
syncNoise.run();
// filler
@@ -285,26 +286,30 @@ public final class SettingsDialog extends JDialog {
c.gridy = row;
c.gridwidth = 2;
c.insets = new Insets(10, 4, 2, 4);
p.add(new JLabel("Input level (speak to test):"), c);
p.add(new JLabel("Input level:"), c);
c.gridy = ++row;
p.add(meter, c);
c.insets = new Insets(4, 4, 4, 4);
c.gridy = ++row;
p.add(buildTestControls(), c);
c.gridwidth = 1;
row++;
vadModeCombo = new JComboBox<>(new String[]{"Automatic", "Volume Gate", "Hybrid"});
vadModeCombo.setSelectedIndex(vadModeIndex(settings.vadMode));
vadModeCombo.addActionListener(e -> micTest.configure(m -> m.setVadMode(currentVadMode())));
vadModeCombo.setToolTipText("<html><b>Automatic</b>: intelligent speech detection.<br>"
+ "<b>Volume Gate</b>: transmit when loud enough.<br>"
+ "<b>Hybrid</b>: loud enough <i>and</i> detected as speech.</html>");
addRow(p, c, row++, new JLabel("Detection:"), vadModeCombo);
thresholdSlider = new JSlider(-70, 0, (int) Math.round(settings.vadThresholdDb));
thresholdSlider = new JSlider((int) InputLevel.MIN_DB, (int) InputLevel.MAX_DB,
(int) Math.round(settings.vadThresholdDb));
thresholdLabel = new JLabel(Math.round(settings.vadThresholdDb) + " dB");
thresholdSlider.addChangeListener(e -> {
meter.setThreshold(thresholdSlider.getValue());
thresholdLabel.setText(thresholdSlider.getValue() + " dB");
if (liveMic != null) liveMic.setThresholdDb(thresholdSlider.getValue());
applyLive(m -> m.setThresholdDb(thresholdSlider.getValue()));
});
addRow(p, c, row++, new JLabel("Volume gate:"), sliderWithLabel(thresholdSlider, thresholdLabel));
@@ -312,7 +317,7 @@ public final class SettingsDialog extends JDialog {
speechLabel = new JLabel(Math.round(settings.speechThreshold * 100) + "%");
speechSlider.addChangeListener(e -> {
speechLabel.setText(speechSlider.getValue() + "%");
if (liveMic != null) liveMic.setSpeechThreshold(speechSlider.getValue() / 100.0);
applyLive(m -> m.setSpeechThreshold(speechSlider.getValue() / 100.0));
});
addRow(p, c, row++, new JLabel("Speech threshold:"), sliderWithLabel(speechSlider, speechLabel));
@@ -322,6 +327,8 @@ public final class SettingsDialog extends JDialog {
addRow(p, c, row++, new JLabel("Push-to-talk hotkey:"), pttKeyButton);
vadOverPttCheck = new JCheckBox("Also detect voice while push-to-talk", settings.vadOverPtt);
vadOverPttCheck.addActionListener(e ->
micTest.configure(m -> m.setVadOverPtt(vadOverPttCheck.isSelected())));
c.gridx = 0;
c.gridy = row++;
c.gridwidth = 2;
@@ -542,46 +549,67 @@ public final class SettingsDialog extends JDialog {
hotkeysPanel.reload();
}
private void apply() {
settings.inputDevice = comboValue(inputCombo);
settings.outputDevice = comboValue(outputCombo);
settings.inputVolume = inputGain.getValue() / 100.0;
settings.outputVolume = outputVol.getValue() / 100.0;
settings.denoise = denoiseCheck.isSelected();
settings.denoiserLevel = denoiseLevel.getValue() / 100.0;
settings.typingAttenuation = typingCheck.isSelected();
settings.agc = agcCheck.isSelected();
settings.inputMode = pttRadio.isSelected() ? Settings.InputMode.PUSH_TO_TALK
/** Copies the audio form into {@code target}, without touching anything else. */
private void writeAudioSettings(Settings target) {
target.inputDevice = comboValue(inputCombo);
target.outputDevice = comboValue(outputCombo);
target.inputVolume = inputGain.getValue() / 100.0;
target.outputVolume = outputVol.getValue() / 100.0;
target.denoise = denoiseCheck.isSelected();
target.denoiserLevel = denoiseLevel.getValue() / 100.0;
target.typingAttenuation = typingCheck.isSelected();
target.agc = agcCheck.isSelected();
target.inputMode = pttRadio.isSelected() ? Settings.InputMode.PUSH_TO_TALK
: contRadio.isSelected() ? Settings.InputMode.CONTINUOUS
: Settings.InputMode.VOICE_ACTIVATION;
settings.vadMode = currentVadMode();
settings.vadThresholdDb = thresholdSlider.getValue();
settings.speechThreshold = speechSlider.getValue() / 100.0;
settings.vadOverPtt = vadOverPttCheck.isSelected();
settings.bitrate = bitrateSlider.getValue() * 1000;
settings.complexity = complexitySlider.getValue();
settings.vbr = vbrCheck.isSelected();
settings.fec = fecCheck.isSelected();
settings.music = musicCheck.isSelected();
target.vadMode = currentVadMode();
target.vadThresholdDb = thresholdSlider.getValue();
target.speechThreshold = speechSlider.getValue() / 100.0;
target.vadOverPtt = vadOverPttCheck.isSelected();
target.bitrate = bitrateSlider.getValue() * 1000;
target.complexity = complexitySlider.getValue();
target.vbr = vbrCheck.isSelected();
target.fec = fecCheck.isSelected();
target.music = musicCheck.isSelected();
}
/**
* The audio form as a standalone {@link Settings}, so the test chain runs with the
* values currently on screen rather than the ones last saved.
*
* <p>The test always runs voice activation: it exists to tune the gate, and push-to-talk
* would need the global hotkey, which belongs to the connected microphone.
*/
private Settings audioSnapshot() {
Settings snapshot = new Settings();
writeAudioSettings(snapshot);
if (snapshot.inputMode == Settings.InputMode.PUSH_TO_TALK) {
snapshot.inputMode = Settings.InputMode.VOICE_ACTIVATION;
}
return snapshot;
}
private void apply() {
writeAudioSettings(settings);
notificationsPanel.apply();
iconPackPanel.apply();
clientVersionPanel.apply();
hotkeysPanel.apply();
settings.save();
if (liveMic != null) {
liveMic.setMode(settings.inputMode);
liveMic.setVadMode(settings.vadMode);
liveMic.setThresholdDb(settings.vadThresholdDb);
liveMic.setSpeechThreshold(settings.speechThreshold);
liveMic.setVadOverPtt(settings.vadOverPtt);
liveMic.setInputGain(settings.inputVolume);
liveMic.setNoiseSuppression(settings.denoise);
liveMic.setDenoiserLevel(settings.denoiserLevel);
liveMic.setTypingAttenuation(settings.typingAttenuation);
liveMic.setAgc(settings.agc);
liveMic.setOpusParameters(OpusParameters.from(settings));
}
applyLive(m -> {
m.setMode(settings.inputMode);
m.setVadMode(settings.vadMode);
m.setThresholdDb(settings.vadThresholdDb);
m.setSpeechThreshold(settings.speechThreshold);
m.setVadOverPtt(settings.vadOverPtt);
m.setInputGain(settings.inputVolume);
m.setNoiseSuppression(settings.denoise);
m.setDenoiserLevel(settings.denoiserLevel);
m.setTypingAttenuation(settings.typingAttenuation);
m.setAgc(settings.agc);
m.setOpusParameters(OpusParameters.from(settings));
});
if (livePlayback != null) {
livePlayback.setMasterVolume(settings.outputVolume);
livePlayback.setOutputDevice(settings.outputDevice);
@@ -597,64 +625,91 @@ public final class SettingsDialog extends JDialog {
}
private void close() {
stopMeter();
micTest.stop();
dispose();
}
// ---- live meter ----
// ---- microphone test ----
private void startMeter() {
meterRunning = true;
meterThread = new Thread(this::meterLoop, "settings-meter");
meterThread.setDaemon(true);
meterThread.start();
/**
* The test row: a toggle that runs the capture chain, an indicator showing whether the
* gate is open, and an optional loopback so you can hear what is being sent.
*/
private JPanel buildTestControls() {
JPanel row = new JPanel(new java.awt.FlowLayout(java.awt.FlowLayout.LEFT, 8, 0));
testButton = new JToggleButton("Begin Test");
testButton.setToolTipText("Run the capture chain exactly as it runs while connected, "
+ "so the bar and the indicator show what would actually be transmitted.");
testButton.addActionListener(e -> setTesting(testButton.isSelected()));
loopbackCheck = new JCheckBox("Hear myself");
loopbackCheck.setToolTipText("Play the transmitted audio back through the playback "
+ "device. Use headphones to avoid feedback.");
loopbackCheck.setEnabled(false);
loopbackCheck.addActionListener(e -> micTest.setLoopback(loopbackCheck.isSelected()));
talkIndicator = new JLabel("Not transmitting", Icons.clientIdle(), JLabel.LEFT);
talkIndicator.setToolTipText("Lights up while your microphone is open, exactly as "
+ "other users would see you in the channel list.");
row.add(testButton);
row.add(loopbackCheck);
row.add(talkIndicator);
return row;
}
private void restartMeter() {
stopMeter();
startMeter();
private void setTesting(boolean on) {
if (on && !micTest.start(audioSnapshot())) {
testButton.setSelected(false);
testButton.setText("Begin Test");
loopbackCheck.setEnabled(false);
resetTestIndicators();
talkIndicator.setText("Capture device unavailable");
return;
}
if (!on) {
micTest.stop();
loopbackCheck.setSelected(false);
resetTestIndicators();
}
testButton.setText(on ? "Stop Test" : "Begin Test");
loopbackCheck.setEnabled(on);
}
private void stopMeter() {
meterRunning = false;
if (meterThread != null) {
meterThread.interrupt();
meterThread = null;
private void resetTestIndicators() {
onTestTalking(false);
onTestLevel(InputLevel.SILENCE_DB);
}
/** Restarts the test chain, if running, so a device change takes effect. */
private void restartTest() {
if (micTest.isRunning()) {
boolean loopback = loopbackCheck.isSelected();
if (micTest.start(audioSnapshot())) {
micTest.setLoopback(loopback);
} else {
setTesting(false);
testButton.setSelected(false);
}
}
}
private void meterLoop() {
String device = comboValue(inputCombo);
AudioCapture line = null;
try {
line = AudioDevices.openCapture(device, AudioDevices.MAX_CHANNELS);
line.start();
int frame = AudioDevices.FRAME_SIZE;
int channels = line.channels();
byte[] buf = new byte[frame * 2 * channels];
double gain = (inputGain != null ? inputGain.getValue() / 100.0 : 1.0);
while (meterRunning) {
if (line.read(buf, 0, buf.length) < buf.length) break;
// Metering follows the capture chain: level is taken off the downmix.
double sumSq = 0;
for (int i = 0; i < frame; i++) {
double mono = 0;
for (int c = 0; c < channels; c++) {
int k = i * channels + c;
short s = (short) ((buf[2 * k + 1] << 8) | (buf[2 * k] & 0xFF));
mono += s / 32768.0;
}
double f = mono / channels * gain;
sumSq += f * f;
}
double rms = Math.sqrt(sumSq / frame);
double db = rms <= 1e-9 ? -100 : 20 * Math.log10(rms);
final double fdb = db;
if (meter != null) SwingUtilities.invokeLater(() -> meter.setLevel(fdb));
}
} catch (Exception ignored) {
} finally {
if (line != null) line.close();
/** Applies a live change to the connected microphone and to the test one alike. */
private void applyLive(java.util.function.Consumer<VoiceInput> change) {
if (liveMic != null) change.accept(liveMic);
micTest.configure(change);
}
private void onTestLevel(double db) {
if (meter != null) meter.setLevel(db);
}
private void onTestTalking(boolean talking) {
if (meter != null) meter.setTransmitting(talking);
if (talkIndicator != null) {
talkIndicator.setIcon(talking ? Icons.clientTalking() : Icons.clientIdle());
talkIndicator.setText(talking ? "Transmitting" : "Not transmitting");
}
}