Replace RMS voice detection with TeamSpeak's own RNN VAD
Ports WebRTC's rnn_vad (as TS3 embeds it) to Java: LPC, pitch estimation, spectral features and the RNN itself, feeding a speech-probability detector that replaces the old SpeechDetector. Also switches the volume-gate threshold from raw dBFS to InputLevel's scale, matching TS3's own slider and range, with a migration for settings saved under the old key.
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
package com.ts3client.audio.vad;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Random;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class RealFftTest {
|
||||
|
||||
/**
|
||||
* The rnn_vad transform length is 480 = 2^5 * 3 * 5, so the mixed-radix path has to be
|
||||
* right for factors other than powers of two.
|
||||
*/
|
||||
@Test
|
||||
void matchesDirectDftForMixedRadixLengths() {
|
||||
for (int n : new int[]{480, 240, 512, 64, 60, 210, 100}) {
|
||||
RealFft fft = new RealFft(n);
|
||||
Random random = new Random(n);
|
||||
float[] in = new float[n];
|
||||
float[] out = new float[n];
|
||||
for (int i = 0; i < n; i++) {
|
||||
in[i] = (float) random.nextGaussian();
|
||||
}
|
||||
fft.forward(in, out);
|
||||
|
||||
for (int k = 0; k <= n / 2; k++) {
|
||||
double re = 0;
|
||||
double im = 0;
|
||||
for (int t = 0; t < n; t++) {
|
||||
double angle = -2 * Math.PI * k * t / n;
|
||||
re += in[t] * Math.cos(angle);
|
||||
im += in[t] * Math.sin(angle);
|
||||
}
|
||||
double gotRe;
|
||||
double gotIm;
|
||||
if (k == 0) {
|
||||
gotRe = out[0];
|
||||
gotIm = 0;
|
||||
} else if (k == n / 2) {
|
||||
gotRe = out[1]; // Packed layout keeps Nyquist in slot 1.
|
||||
gotIm = 0;
|
||||
} else {
|
||||
gotRe = out[2 * k];
|
||||
gotIm = out[2 * k + 1];
|
||||
}
|
||||
assertEquals(re, gotRe, 1e-2, "n=" + n + " bin=" + k + " real");
|
||||
assertEquals(im, gotIm, 1e-2, "n=" + n + " bin=" + k + " imag");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsOddLengths() {
|
||||
org.junit.jupiter.api.Assertions.assertThrows(
|
||||
IllegalArgumentException.class, () -> new RealFft(481));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package com.ts3client.audio.vad;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Random;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Behavioural checks on the {@code rnn_vad} port. The network weights are the ones the TS3
|
||||
* client links, so these also pin the detector's agreement with it.
|
||||
*/
|
||||
class RnnVadTest {
|
||||
|
||||
private static final int FRAMES = 60;
|
||||
|
||||
@Test
|
||||
void silenceScoresZero() {
|
||||
float[] x = new float[RnnVad.FRAME_SIZE * FRAMES];
|
||||
double[] probabilities = run(x);
|
||||
for (double p : probabilities) {
|
||||
assertEquals(0.0, p, 0.0, "silence must not score above zero");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void voicedSpeechLikeSignalScoresHigh() {
|
||||
float[] x = synth(FRAMES, t -> {
|
||||
double sum = 0;
|
||||
for (int h = 1; h <= 30 && 120 * h < 8000; h++) {
|
||||
double f = 120 * h;
|
||||
double gain = formant(f, 700, 90) + formant(f, 1220, 110) + formant(f, 2600, 160);
|
||||
sum += gain * Math.sin(2 * Math.PI * f * t + h);
|
||||
}
|
||||
// A slow tremolo keeps it from looking like a stationary tone.
|
||||
return 0.25 * (0.6 + 0.4 * Math.sin(2 * Math.PI * 5 * t)) * sum / 6.0;
|
||||
});
|
||||
// Skip the first frames: the pitch buffer and GRU need history.
|
||||
assertTrue(meanFrom(run(x), 20) > 0.8, "a voiced signal should score high");
|
||||
}
|
||||
|
||||
@Test
|
||||
void quietPinkNoiseScoresLow() {
|
||||
Random random = new Random(3);
|
||||
float[] x = new float[RnnVad.FRAME_SIZE * FRAMES];
|
||||
double b0 = 0;
|
||||
double b1 = 0;
|
||||
double b2 = 0;
|
||||
for (int i = 0; i < x.length; i++) {
|
||||
double w = random.nextGaussian();
|
||||
b0 = 0.99765 * b0 + w * 0.0990460;
|
||||
b1 = 0.96300 * b1 + w * 0.2965164;
|
||||
b2 = 0.57000 * b2 + w * 1.0526913;
|
||||
// About -45 dBFS: representative of room tone rather than loud hiss.
|
||||
x[i] = (float) ((b0 + b1 + b2 + w * 0.1848) * 0.0011);
|
||||
}
|
||||
assertTrue(meanFrom(run(x), 20) < 0.2, "quiet room tone should not read as speech");
|
||||
}
|
||||
|
||||
@Test
|
||||
void estimatesPitchPeriodExactly() {
|
||||
// The pitch feature is fed to the network as 0.01 * (period48k - 300).
|
||||
for (double f0 : new double[]{80, 100, 120, 150, 200, 250, 300, 400}) {
|
||||
RnnVad vad = new RnnVad();
|
||||
float[] x = synth(FRAMES, t -> {
|
||||
double sum = 0;
|
||||
for (int h = 1; h <= 40 && f0 * h < 11000; h++) {
|
||||
sum += Math.sin(2 * Math.PI * f0 * h * t) / h;
|
||||
}
|
||||
return 0.2 * sum;
|
||||
});
|
||||
for (int f = 0; f < FRAMES; f++) {
|
||||
vad.process(x, f * RnnVad.FRAME_SIZE);
|
||||
}
|
||||
assertEquals(48000 / f0, vad.pitchPeriod48kHz(), 1e-9,
|
||||
"pitch period for f0=" + f0);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void resetClearsState() {
|
||||
float[] x = synth(FRAMES, t -> 0.3 * Math.sin(2 * Math.PI * 200 * t));
|
||||
RnnVad vad = new RnnVad();
|
||||
for (int f = 0; f < FRAMES; f++) {
|
||||
vad.process(x, f * RnnVad.FRAME_SIZE);
|
||||
}
|
||||
assertTrue(vad.probability() > 0.0);
|
||||
vad.reset();
|
||||
assertEquals(0.0, vad.probability(), 0.0);
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
private interface Signal {
|
||||
double at(double t);
|
||||
}
|
||||
|
||||
private static double formant(double f, double centre, double bandwidth) {
|
||||
double d = (f - centre) / bandwidth;
|
||||
return 1.0 / (1.0 + d * d);
|
||||
}
|
||||
|
||||
private static float[] synth(int frames, Signal signal) {
|
||||
float[] x = new float[RnnVad.FRAME_SIZE * frames];
|
||||
for (int i = 0; i < x.length; i++) {
|
||||
x[i] = (float) signal.at((double) i / RnnVad.SAMPLE_RATE);
|
||||
}
|
||||
return x;
|
||||
}
|
||||
|
||||
private static double[] run(float[] x) {
|
||||
RnnVad vad = new RnnVad();
|
||||
int frames = x.length / RnnVad.FRAME_SIZE;
|
||||
double[] probabilities = new double[frames];
|
||||
for (int i = 0; i < frames; i++) {
|
||||
probabilities[i] = vad.process(x, i * RnnVad.FRAME_SIZE);
|
||||
}
|
||||
return probabilities;
|
||||
}
|
||||
|
||||
private static double meanFrom(double[] values, int start) {
|
||||
double sum = 0;
|
||||
for (int i = start; i < values.length; i++) {
|
||||
sum += values[i];
|
||||
}
|
||||
return sum / (values.length - start);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user