The array is reused between calls, so copy anything you need to keep. Called from + * the audio thread: do not block. + * + * @param interleaved samples, {@code channels} per sample position + * @param channels number of interleaved channels + */ + void onFrame(float[] interleaved, int channels); +} diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/InputLevel.java b/ts3-client/core/src/main/java/com/ts3client/audio/InputLevel.java new file mode 100644 index 0000000..905a093 --- /dev/null +++ b/ts3-client/core/src/main/java/com/ts3client/audio/InputLevel.java @@ -0,0 +1,58 @@ +package com.ts3client.audio; + +/** + * Capture level on the scale the TeamSpeak 3 client puts on its voice-activation slider. + * + *
TS3 stores {@code voiceactivation_level} in decibels but compares powers: its setter + * converts the slider value with {@code 10^(dB/10) * 0.0095} and its getter reports a + * measured power back with {@code 10*log10(power / 0.0095)}. The same reference constant is + * used here so a threshold of −40 means the same thing it does in TeamSpeak. + * + *
The resulting scale runs from {@link #MIN_DB} to {@link #MAX_DB}, with 0 dB + * corresponding to roughly −20 dBFS and the default −40 to about + * −60 dBFS. + */ +public final class InputLevel { + + /** + * Mean-square level that reads as 0 dB, taken from the client's own dB conversion. + */ + private static final double REFERENCE_POWER = 0.0095; + + /** Lower end of the slider range. */ + public static final double MIN_DB = -50.0; + + /** Upper end of the slider range. */ + public static final double MAX_DB = 50.0; + + /** Reported when the frame is digital silence. */ + public static final double SILENCE_DB = MIN_DB; + + private InputLevel() { + } + + /** Level of {@code length} mono samples in [-1, 1], in dB on the scale above. */ + public static double toDb(float[] frame, int length) { + if (length <= 0) { + return SILENCE_DB; + } + double sumSq = 0; + for (int i = 0; i < length; i++) { + sumSq += (double) frame[i] * frame[i]; + } + return powerToDb(sumSq / length); + } + + /** Converts a mean-square power to the slider's dB scale. */ + public static double powerToDb(double power) { + if (power <= 0) { + return SILENCE_DB; + } + return Math.max(SILENCE_DB, 10.0 * Math.log10(power / REFERENCE_POWER)); + } + + /** Inverse of {@link #powerToDb}, matching the client's threshold conversion. */ + public static double dbToPower(double db) { + return Math.pow(10.0, db / 10.0) * REFERENCE_POWER; + } +} diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/SpeechDetector.java b/ts3-client/core/src/main/java/com/ts3client/audio/SpeechDetector.java deleted file mode 100644 index 360df01..0000000 --- a/ts3-client/core/src/main/java/com/ts3client/audio/SpeechDetector.java +++ /dev/null @@ -1,112 +0,0 @@ -package com.ts3client.audio; - -/** - * Lightweight speech-presence detector used by the "Automatic" and "Hybrid" voice - * activation modes (the TeamSpeak 3 client uses a WebRTC GMM detector for the same - * purpose). It classifies each 20 ms frame from three features — short-term - * energy, spectral flatness and dominant frequency — against an adaptive noise - * floor, following Moattar & Homayounpour's real-time VAD. - * - *
Pure DSP with no platform dependencies, so any frontend/backend can reuse it. - * A frame votes for speech when at least two of the three features exceed the - * baseline; the smoothed vote fraction is exposed as a [0,1] probability. - */ -public final class SpeechDetector { - - private static final int FFT_SIZE = 1024; - - // Primary thresholds from the reference algorithm (16-bit sample scale). - private static final double ENERGY_PRIM = 40.0; - private static final double DOMINANT_FREQ_PRIM = 185.0; - private static final double FLATNESS_PRIM = 5.0; - - private final int sampleRate; - private final double[] re = new double[FFT_SIZE]; - private final double[] im = new double[FFT_SIZE]; - - private double minEnergy; - private double minDominantFreq; - private double minFlatness; - private long silenceFrames; - private long frameCount; - private double probability; - - public SpeechDetector(int sampleRate) { - this.sampleRate = sampleRate; - } - - public void reset() { - frameCount = 0; - silenceFrames = 0; - probability = 0; - } - - public double getProbability() { - return probability; - } - - /** - * Processes one frame of mono PCM in [-1,1] and returns the smoothed speech - * probability in [0,1]. - */ - public double process(float[] frame) { - double energy = 0; - int n = Math.min(frame.length, FFT_SIZE); - for (int i = 0; i < n; i++) { - double s = frame[i] * 32768.0; // emulate 16-bit scale - energy += s * s; - re[i] = frame[i]; - im[i] = 0; - } - for (int i = n; i < FFT_SIZE; i++) { - re[i] = 0; - im[i] = 0; - } - - Fft.forward(re, im); - - int half = FFT_SIZE / 2; - double geoLogSum = 0; - double arithSum = 0; - double maxMag = 0; - int maxBin = 0; - for (int k = 1; k < half; k++) { - double mag = Math.sqrt(re[k] * re[k] + im[k] * im[k]) + 1e-12; - geoLogSum += Math.log(mag); - arithSum += mag; - if (mag > maxMag) { - maxMag = mag; - maxBin = k; - } - } - int bins = half - 1; - double geoMean = Math.exp(geoLogSum / bins); - double arithMean = arithSum / bins; - double flatness = -10.0 * Math.log10(geoMean / arithMean); // high = tonal (speech) - double dominantFreq = (double) maxBin * sampleRate / FFT_SIZE; - - if (frameCount == 0) { - minEnergy = Math.max(energy, 1.0); - minDominantFreq = dominantFreq; - minFlatness = flatness; - } - - double energyThresh = ENERGY_PRIM * Math.log10(Math.max(minEnergy, 1.0)); - int votes = 0; - if (energy - minEnergy >= energyThresh) votes++; - if (dominantFreq - minDominantFreq >= DOMINANT_FREQ_PRIM) votes++; - if (flatness - minFlatness >= FLATNESS_PRIM) votes++; - - boolean speech = votes >= 2; - if (!speech) { - // Adapt the noise floor towards the current (silent) energy. - silenceFrames++; - minEnergy = ((silenceFrames * minEnergy) + energy) / (silenceFrames + 1); - } - - double instant = votes / 3.0; - probability = 0.6 * probability + 0.4 * instant; - frameCount++; - return probability; - } -} diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/SpeechProbabilityDetector.java b/ts3-client/core/src/main/java/com/ts3client/audio/SpeechProbabilityDetector.java new file mode 100644 index 0000000..ddf04aa --- /dev/null +++ b/ts3-client/core/src/main/java/com/ts3client/audio/SpeechProbabilityDetector.java @@ -0,0 +1,27 @@ +package com.ts3client.audio; + +/** + * Estimates how likely it is that a frame of captured audio contains speech. + * + *
Used by the "Automatic" and "Hybrid" voice-activation modes. Implementations are + * stateful and expect frames in capture order; {@link #reset()} clears that state. + * + *
Pure DSP with no platform dependencies, so any frontend or backend can reuse it.
+ */
+public interface SpeechProbabilityDetector {
+
+ /**
+ * Consumes one frame of mono PCM in [-1, 1] and returns the current speech
+ * probability in [0, 1].
+ *
+ * @param frame samples in capture order
+ * @param length number of valid samples in {@code frame}
+ */
+ double process(float[] frame, int length);
+
+ /** Speech probability from the most recent {@link #process} call. */
+ double probability();
+
+ /** Drops all history. Call on a capture discontinuity or a settings change. */
+ void reset();
+}
diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceInput.java b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceInput.java
index 5e18c3f..3312523 100644
--- a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceInput.java
+++ b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceInput.java
@@ -22,7 +22,7 @@ public interface VoiceInput extends Microphone {
void setVadMode(Settings.VadMode mode);
- /** Voice-activation volume-gate threshold in dBFS. */
+ /** Volume-gate threshold on {@link InputLevel}'s scale. */
void setThresholdDb(double db);
/** Speech-probability threshold (0..1) for Automatic/Hybrid modes. */
@@ -50,12 +50,19 @@ public interface VoiceInput extends Microphone {
void setPushToTalk(boolean pressed);
- /** Receives the live input level in dBFS, once per captured frame. */
+ /** Receives the live input level on {@link InputLevel}'s scale, once per captured frame. */
void setLevelListener(Consumer Whitening the signal before searching for a pitch makes the auto-correlation peak at
+ * the true period rather than at whichever formant happens to dominate.
+ */
+final class LinearPrediction {
+
+ private final float[] autoCorr = new float[NUM_LPC_COEFFICIENTS];
+ private final float[] reflection = new float[NUM_LPC_COEFFICIENTS - 1];
+
+ /** Computes the 5 post-processed inverse-filter coefficients for {@code x}. */
+ void computeLpcCoefficients(float[] x, float[] lpcCoeffs) {
+ for (int lag = 0; lag < NUM_LPC_COEFFICIENTS; lag++) {
+ float sum = 0;
+ for (int i = 0; i < x.length - lag; i++) {
+ sum += x[i] * x[i + lag];
+ }
+ autoCorr[lag] = sum;
+ }
+ if (autoCorr[0] == 0.0f) { // Empty frame.
+ java.util.Arrays.fill(lpcCoeffs, 0.0f);
+ return;
+ }
+
+ // Assume a -40 dB white noise floor.
+ autoCorr[0] *= 1.0001f;
+ autoCorr[1] -= autoCorr[1] * 0.000064f;
+ autoCorr[2] -= autoCorr[2] * 0.000256f;
+ autoCorr[3] -= autoCorr[3] * 0.000576f;
+ autoCorr[4] -= autoCorr[4] * 0.001024f;
+
+ java.util.Arrays.fill(reflection, 0.0f);
+ computeInitialInverseFilterCoefficients();
+
+ // Bandwidth expansion, then convolution with (1 + 0.8 z^-1).
+ reflection[0] *= 0.9f;
+ reflection[1] *= 0.9f * 0.9f;
+ reflection[2] *= 0.9f * 0.9f * 0.9f;
+ reflection[3] *= 0.9f * 0.9f * 0.9f * 0.9f;
+ final float c = 0.8f;
+ lpcCoeffs[0] = reflection[0] + c;
+ lpcCoeffs[1] = reflection[1] + c * reflection[0];
+ lpcCoeffs[2] = reflection[2] + c * reflection[1];
+ lpcCoeffs[3] = reflection[3] + c * reflection[2];
+ lpcCoeffs[4] = c * reflection[3];
+ }
+
+ /** Levinson–Durbin recursion over {@link #autoCorr}, writing {@link #reflection}. */
+ private void computeInitialInverseFilterCoefficients() {
+ float error = autoCorr[0];
+ for (int i = 0; i < NUM_LPC_COEFFICIENTS - 1; i++) {
+ float reflectionCoeff = 0;
+ for (int j = 0; j < i; j++) {
+ reflectionCoeff += reflection[j] * autoCorr[i - j];
+ }
+ reflectionCoeff += autoCorr[i + 1];
+
+ // Avoid dividing by numbers close to zero.
+ final float minErrorMagnitude = 1e-6f;
+ if (Math.abs(error) < minErrorMagnitude) {
+ error = Math.copySign(minErrorMagnitude, error);
+ }
+ reflectionCoeff /= -error;
+
+ reflection[i] = reflectionCoeff;
+ for (int j = 0; j < ((i + 1) >> 1); j++) {
+ float tmp1 = reflection[j];
+ float tmp2 = reflection[i - 1 - j];
+ reflection[j] = tmp1 + reflectionCoeff * tmp2;
+ reflection[i - 1 - j] = tmp2 + reflectionCoeff * tmp1;
+ }
+ error -= reflectionCoeff * reflectionCoeff * error;
+ if (error < 0.001f * autoCorr[0]) {
+ break;
+ }
+ }
+ }
+
+ /**
+ * Applies the inverse filter: {@code y[i] = x[i] + sum_j lpc[j] * x[i - 1 - j]},
+ * with the sum truncated near the start of the buffer.
+ */
+ static void computeLpResidual(float[] lpcCoeffs, float[] x, float[] y) {
+ for (int i = 0; i < x.length; i++) {
+ float sum = x[i];
+ int taps = Math.min(i, NUM_LPC_COEFFICIENTS);
+ for (int j = 0; j < taps; j++) {
+ sum += lpcCoeffs[j] * x[i - 1 - j];
+ }
+ y[i] = sum;
+ }
+ }
+}
diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/vad/PitchEstimator.java b/ts3-client/core/src/main/java/com/ts3client/audio/vad/PitchEstimator.java
new file mode 100644
index 0000000..6fe4a60
--- /dev/null
+++ b/ts3-client/core/src/main/java/com/ts3client/audio/vad/PitchEstimator.java
@@ -0,0 +1,329 @@
+package com.ts3client.audio.vad;
+
+import static com.ts3client.audio.vad.VadConstants.BUF_SIZE_12K;
+import static com.ts3client.audio.vad.VadConstants.FRAME_20MS_12K;
+import static com.ts3client.audio.vad.VadConstants.FRAME_20MS_24K;
+import static com.ts3client.audio.vad.VadConstants.INITIAL_NUM_LAGS_24K;
+import static com.ts3client.audio.vad.VadConstants.MAX_PITCH_24K;
+import static com.ts3client.audio.vad.VadConstants.MAX_PITCH_48K;
+import static com.ts3client.audio.vad.VadConstants.MIN_PITCH_24K;
+import static com.ts3client.audio.vad.VadConstants.MIN_PITCH_48K;
+import static com.ts3client.audio.vad.VadConstants.NUM_LAGS_12K;
+import static com.ts3client.audio.vad.VadConstants.REFINE_NUM_LAGS_24K;
+
+/**
+ * Pitch period estimation for {@code rnn_vad}, ported from WebRTC's
+ * {@code rnn_vad/pitch_search.cc} and {@code pitch_search_internal.cc}.
+ *
+ * Three stages: a coarse search over the 12 kHz decimated LP residual picks two
+ * candidates, those are refined against the 24 kHz buffer, and finally sub-harmonics
+ * of the winner are tested so that an octave error can be corrected. The result is a
+ * period expressed at 48 kHz, which is what the network was trained on.
+ *
+ * Lags are stored inverted: index 0 corresponds to the maximum pitch period.
+ */
+final class PitchEstimator {
+
+ /**
+ * For each divisor k in [2, 15], the multiplier n such that n*T/k is a sub-harmonic
+ * worth checking alongside T/k, chosen so no multiple is visited twice.
+ */
+ private static final int[] SUB_HARMONIC_MULTIPLIERS =
+ {3, 2, 3, 2, 5, 2, 3, 2, 3, 2, 5, 2, 3, 2};
+
+ /** Thresholds on the initial period, computed as {@code [5*k*k for k in range(16)]}. */
+ private static final int[] INITIAL_PITCH_PERIOD_THRESHOLDS =
+ {20, 45, 80, 125, 180, 245, 320, 405, 500, 605, 720, 845, 980, 1125};
+
+ /** Number of pitch periods analysed either side of a candidate during refinement. */
+ private static final int PITCH_NEIGHBORHOOD_RADIUS = 2;
+
+ private final float[] pitchBuffer12k = new float[BUF_SIZE_12K];
+ private final float[] autoCorrelation12k = new float[NUM_LAGS_12K];
+ private final float[] yEnergy24k = new float[REFINE_NUM_LAGS_24K];
+ private final float[] autoCorrelation24k = new float[INITIAL_NUM_LAGS_24K];
+ private final int[] invertedLagsIndex = new int[2 * (2 * PITCH_NEIGHBORHOOD_RADIUS + 1)];
+ private int invertedLagsCount;
+
+ // Pitch tracking: the previous estimate biases the sub-harmonic decision.
+ private int lastPeriod48k;
+ private float lastStrength;
+
+ void reset() {
+ lastPeriod48k = 0;
+ lastStrength = 0;
+ }
+
+ /** Returns the pitch period at 48 kHz for the given 24 kHz LP residual buffer. */
+ int estimate(float[] pitchBuffer) {
+ // Coarse search at 12 kHz. WebRTC decimates without an anti-aliasing filter here;
+ // the refinement stage below is what recovers the accuracy.
+ for (int i = 0; i < BUF_SIZE_12K; i++) {
+ pitchBuffer12k[i] = pitchBuffer[2 * i];
+ }
+ computeAutoCorrelation12k();
+ long candidates = computePitchPeriod12kHz();
+ int best = (int) (candidates >> 32) * 2;
+ int secondBest = (int) candidates * 2;
+
+ computeSlidingFrameSquareEnergies24k(pitchBuffer);
+ int pitchLag48k = computePitchPeriod48kHz(pitchBuffer, best, secondBest);
+ computeExtendedPitchPeriod48kHz(pitchBuffer, MAX_PITCH_48K - pitchLag48k);
+ return lastPeriod48k;
+ }
+
+ /**
+ * {@code autoCorrelation12k[lag]} is the correlation of the most recent 20 ms with the
+ * frame starting {@code lag} samples into the buffer.
+ */
+ private void computeAutoCorrelation12k() {
+ final int refOffset = BUF_SIZE_12K - FRAME_20MS_12K;
+ for (int lag = 0; lag < NUM_LAGS_12K; lag++) {
+ float sum = 0;
+ for (int i = 0; i < FRAME_20MS_12K; i++) {
+ sum += pitchBuffer12k[refOffset + i] * pitchBuffer12k[lag + i];
+ }
+ autoCorrelation12k[lag] = sum;
+ }
+ }
+
+ /** Returns the best and second-best inverted lags packed into a long. */
+ private long computePitchPeriod12kHz() {
+ float denominator = 1.0f;
+ for (int i = 0; i < FRAME_20MS_12K + 1; i++) {
+ denominator += pitchBuffer12k[i] * pitchBuffer12k[i];
+ }
+
+ int bestLag = 0;
+ float bestNumerator = -1.0f;
+ float bestDenominator = 0.0f;
+ int secondLag = 1;
+ float secondNumerator = -1.0f;
+ float secondDenominator = 0.0f;
+
+ for (int invertedLag = 0; invertedLag < NUM_LAGS_12K; invertedLag++) {
+ // A pitch candidate must have positive correlation.
+ if (autoCorrelation12k[invertedLag] > 0.0f) {
+ float numerator = autoCorrelation12k[invertedLag] * autoCorrelation12k[invertedLag];
+ // Compare numerator/denominator ratios without dividing.
+ if (numerator * secondDenominator > secondNumerator * denominator) {
+ if (numerator * bestDenominator > bestNumerator * denominator) {
+ secondLag = bestLag;
+ secondNumerator = bestNumerator;
+ secondDenominator = bestDenominator;
+ bestLag = invertedLag;
+ bestNumerator = numerator;
+ bestDenominator = denominator;
+ } else {
+ secondLag = invertedLag;
+ secondNumerator = numerator;
+ secondDenominator = denominator;
+ }
+ }
+ }
+ // Slide the energy window to the next inverted lag.
+ float yOld = pitchBuffer12k[invertedLag];
+ float yNew = pitchBuffer12k[invertedLag + FRAME_20MS_12K];
+ denominator -= yOld * yOld;
+ denominator += yNew * yNew;
+ denominator = Math.max(0.0f, denominator);
+ }
+ return ((long) bestLag << 32) | (secondLag & 0xffffffffL);
+ }
+
+ /** Energy of the sliding 20 ms frame for every inverted lag. */
+ private void computeSlidingFrameSquareEnergies24k(float[] pitchBuffer) {
+ float yy = 0;
+ for (int i = 0; i < FRAME_20MS_24K; i++) {
+ yy += pitchBuffer[i] * pitchBuffer[i];
+ }
+ yEnergy24k[0] = yy;
+ for (int invertedLag = 0; invertedLag < MAX_PITCH_24K; invertedLag++) {
+ yy -= pitchBuffer[invertedLag] * pitchBuffer[invertedLag];
+ yy += pitchBuffer[invertedLag + FRAME_20MS_24K] * pitchBuffer[invertedLag + FRAME_20MS_24K];
+ yy = Math.max(1.0f, yy);
+ yEnergy24k[invertedLag + 1] = yy;
+ }
+ }
+
+ /** Correlation of the most recent 20 ms with the frame at {@code invertedLag}. */
+ private static float autoCorrelation(int invertedLag, float[] pitchBuffer) {
+ float sum = 0;
+ for (int i = 0; i < FRAME_20MS_24K; i++) {
+ sum += pitchBuffer[MAX_PITCH_24K + i] * pitchBuffer[invertedLag + i];
+ }
+ return sum;
+ }
+
+ /**
+ * Recomputes the auto-correlation around both candidates at 24 kHz and returns the
+ * strongest inverted lag, pseudo-interpolated to 48 kHz.
+ */
+ private int computePitchPeriod48kHz(float[] pitchBuffer, int bestCandidate, int secondCandidate) {
+ invertedLagsCount = 0;
+ // Order the two neighbourhoods so the first precedes the second.
+ boolean swap = bestCandidate > secondCandidate;
+ int firstCenter = swap ? secondCandidate : bestCandidate;
+ int secondCenter = swap ? bestCandidate : secondCandidate;
+ int r1min = Math.max(firstCenter - PITCH_NEIGHBORHOOD_RADIUS, 0);
+ int r1max = Math.min(firstCenter + PITCH_NEIGHBORHOOD_RADIUS, INITIAL_NUM_LAGS_24K - 1);
+ int r2min = Math.max(secondCenter - PITCH_NEIGHBORHOOD_RADIUS, 0);
+ int r2max = Math.min(secondCenter + PITCH_NEIGHBORHOOD_RADIUS, INITIAL_NUM_LAGS_24K - 1);
+
+ if (r1max + 1 >= r2min) { // Overlapping or adjacent ranges.
+ computeAutoCorrelationRange(pitchBuffer, r1min, r2max);
+ } else {
+ computeAutoCorrelationRange(pitchBuffer, r1min, r1max);
+ computeAutoCorrelationRange(pitchBuffer, r2min, r2max);
+ }
+
+ int bestInvertedLag = 0;
+ float bestNumerator = -1.0f;
+ float bestDenominator = 0.0f;
+ for (int i = 0; i < invertedLagsCount; i++) {
+ int invertedLag = invertedLagsIndex[i];
+ if (autoCorrelation24k[invertedLag] > 0.0f) {
+ float numerator = autoCorrelation24k[invertedLag] * autoCorrelation24k[invertedLag];
+ float denominator = yEnergy24k[invertedLag];
+ if (numerator * bestDenominator > bestNumerator * denominator) {
+ bestInvertedLag = invertedLag;
+ bestNumerator = numerator;
+ bestDenominator = denominator;
+ }
+ }
+ }
+
+ if (bestInvertedLag == 0 || bestInvertedLag >= INITIAL_NUM_LAGS_24K - 1) {
+ return bestInvertedLag * 2; // Cannot interpolate at the boundaries.
+ }
+ int offset = pseudoInterpolationOffset(
+ autoCorrelation24k[bestInvertedLag + 1],
+ autoCorrelation24k[bestInvertedLag],
+ autoCorrelation24k[bestInvertedLag - 1]);
+ return 2 * bestInvertedLag + offset;
+ }
+
+ private void computeAutoCorrelationRange(float[] pitchBuffer, int min, int max) {
+ // The pseudo-interpolation reads one lag either side, so zero those rather than
+ // clearing the whole array.
+ if (min > 0) {
+ autoCorrelation24k[min - 1] = 0.0f;
+ }
+ if (max < INITIAL_NUM_LAGS_24K - 1) {
+ autoCorrelation24k[max + 1] = 0.0f;
+ }
+ for (int invertedLag = min; invertedLag <= max; invertedLag++) {
+ autoCorrelation24k[invertedLag] = autoCorrelation(invertedLag, pitchBuffer);
+ invertedLagsIndex[invertedLagsCount++] = invertedLag;
+ }
+ }
+
+ /** Returns a lag correction in {-1, 0, +1} based on the neighbouring correlations. */
+ private static int pseudoInterpolationOffset(float prev, float curr, float next) {
+ if ((next - prev) > 0.7f * (curr - prev)) {
+ return 1;
+ } else if ((prev - next) > 0.7f * (curr - next)) {
+ return -1;
+ }
+ return 0;
+ }
+
+ /** Refines {@code lag} (24 kHz) to a 48 kHz lag using pseudo-interpolation. */
+ private static int pseudoInterpolationLagPitchBuf(int lag, float[] pitchBuffer) {
+ int offset = 0;
+ if (lag > 0 && lag < MAX_PITCH_24K) {
+ int invertedLag = MAX_PITCH_24K - lag;
+ offset = pseudoInterpolationOffset(
+ autoCorrelation(invertedLag + 1, pitchBuffer),
+ autoCorrelation(invertedLag, pitchBuffer),
+ autoCorrelation(invertedLag - 1, pitchBuffer));
+ }
+ return 2 * lag + offset;
+ }
+
+ /** Same as {@code round(multiplier * pitchPeriod / divisor)}. */
+ private static int alternativePitchPeriod(int pitchPeriod, int multiplier, int divisor) {
+ return (2 * multiplier * pitchPeriod + divisor) / (2 * divisor);
+ }
+
+ /**
+ * Tests sub-harmonics of the refined period and keeps the strongest, then stores the
+ * winner in {@link #lastPeriod48k}/{@link #lastStrength}.
+ */
+ private void computeExtendedPitchPeriod48kHz(float[] pitchBuffer, int initialPeriod48k) {
+ final float xEnergy = yEnergy24k[MAX_PITCH_24K];
+
+ int bestPeriod = Math.min(initialPeriod48k / 2, MAX_PITCH_24K - 1);
+ float bestXy = autoCorrelation(MAX_PITCH_24K - bestPeriod, pitchBuffer);
+ float bestYEnergy = yEnergy24k[MAX_PITCH_24K - bestPeriod];
+ float bestStrength = pitchStrength(bestXy, bestYEnergy, xEnergy);
+
+ final int initialPeriod = bestPeriod;
+ final float initialStrength = bestStrength;
+ // The tracked pitch is kept at 48 kHz but compared at 24 kHz.
+ final int lastPeriod = lastPeriod48k / 2;
+
+ // Largest divisor for which the alternative period is still above the minimum.
+ int maxPeriodDivisor = (2 * initialPeriod) / (2 * MIN_PITCH_24K - 1);
+ for (int periodDivisor = 2; periodDivisor <= maxPeriodDivisor; periodDivisor++) {
+ int alternativePeriod = alternativePitchPeriod(initialPeriod, 1, periodDivisor);
+ int dualPeriod = alternativePitchPeriod(
+ initialPeriod, SUB_HARMONIC_MULTIPLIERS[periodDivisor - 2], periodDivisor);
+ // Special case: for a divisor of 2 the sub-harmonic can exceed the pitch range.
+ if (periodDivisor == 2 && dualPeriod > MAX_PITCH_24K) {
+ dualPeriod = initialPeriod;
+ }
+
+ // Score the candidate together with its sub-harmonic.
+ float xyPrimary = autoCorrelation(MAX_PITCH_24K - alternativePeriod, pitchBuffer);
+ float xySecondary = autoCorrelation(MAX_PITCH_24K - dualPeriod, pitchBuffer);
+ float xy = 0.5f * (xyPrimary + xySecondary);
+ float yy = 0.5f * (yEnergy24k[MAX_PITCH_24K - alternativePeriod]
+ + yEnergy24k[MAX_PITCH_24K - dualPeriod]);
+ float alternativeStrength = pitchStrength(xy, yy, xEnergy);
+
+ if (isAlternativeStronger(lastPeriod, initialPeriod, initialStrength,
+ alternativePeriod, alternativeStrength, periodDivisor)) {
+ bestPeriod = alternativePeriod;
+ bestStrength = alternativeStrength;
+ bestXy = xy;
+ bestYEnergy = yy;
+ }
+ }
+
+ bestXy = Math.max(0.0f, bestXy);
+ float finalStrength = (bestYEnergy <= bestXy) ? 1.0f : bestXy / (bestYEnergy + 1.0f);
+ finalStrength = Math.min(bestStrength, finalStrength);
+
+ lastPeriod48k = Math.max(MIN_PITCH_48K, pseudoInterpolationLagPitchBuf(bestPeriod, pitchBuffer));
+ lastStrength = finalStrength;
+ }
+
+ private static float pitchStrength(float xy, float yEnergy, float xEnergy) {
+ return xy / (float) Math.sqrt(1.0f + xEnergy * yEnergy);
+ }
+
+ private boolean isAlternativeStronger(int lastPeriod, int initialPeriod, float initialStrength,
+ int alternativePeriod, float alternativeStrength,
+ int periodDivisor) {
+ // Pitch tracking: a candidate close to the previous estimate clears a lower bar.
+ float lowerThresholdTerm = 0.0f;
+ int distance = Math.abs(alternativePeriod - lastPeriod);
+ if (distance <= 1) {
+ lowerThresholdTerm = lastStrength;
+ } else if (distance == 2
+ && initialPeriod > INITIAL_PITCH_PERIOD_THRESHOLDS[periodDivisor - 2]) {
+ lowerThresholdTerm = 0.5f * lastStrength;
+ }
+
+ // Higher frequencies get a stricter threshold: short-term correlations bias
+ // the search towards them and cause false positives.
+ float threshold = Math.max(0.3f, 0.7f * initialStrength - lowerThresholdTerm);
+ if (alternativePeriod < 3 * MIN_PITCH_24K) {
+ threshold = Math.max(0.4f, 0.85f * initialStrength - lowerThresholdTerm);
+ } else if (alternativePeriod < 2 * MIN_PITCH_24K) {
+ threshold = Math.max(0.5f, 0.9f * initialStrength - lowerThresholdTerm);
+ }
+ return alternativeStrength > threshold;
+ }
+}
diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/vad/RealFft.java b/ts3-client/core/src/main/java/com/ts3client/audio/vad/RealFft.java
new file mode 100644
index 0000000..e1ce6bb
--- /dev/null
+++ b/ts3-client/core/src/main/java/com/ts3client/audio/vad/RealFft.java
@@ -0,0 +1,224 @@
+package com.ts3client.audio.vad;
+
+/**
+ * Real-input FFT for the {@code rnn_vad} spectral analysis.
+ *
+ * The transform length is 480 (a 20 ms frame at 24 kHz), which is not a
+ * power of two, so the shared radix-2 {@code Fft} cannot be used. This is a
+ * mixed-radix Cooley–Tukey complex FFT (radices 2, 3, 4 and 5, with a generic
+ * DFT for any other factor) wrapped in the standard real-input trick: an
+ * {@code n}-point real transform is computed as an {@code n/2}-point complex one
+ * followed by a split step.
+ *
+ * Output uses the packed layout WebRTC's PFFFT produces, which the band
+ * correlator indexes directly:
+ * A transform of length {@code n = r * m} is computed by transforming the {@code r}
+ * interleaved subsequences of length {@code m} and combining them:
+ * {@code X[k1*m + k2] = sum_j (W_n^(j*k2) * X_j[k2]) * W_r^(j*k1)}.
+ *
+ * The recursion is expressed directly rather than as an iterative digit-reversed
+ * loop, which keeps it correct for any factorisation. Scratch buffers are allocated
+ * once per recursion level so no transform allocates.
+ */
+ static final class ComplexFft {
+
+ private final int n;
+ private final float[] cosTable;
+ private final float[] sinTable;
+ private final float[][] scratchRe;
+ private final float[][] scratchIm;
+ private final float[] dftRe;
+ private final float[] dftIm;
+
+ ComplexFft(int n) {
+ this.n = n;
+ this.cosTable = new float[n];
+ this.sinTable = new float[n];
+ for (int i = 0; i < n; i++) {
+ double angle = -2.0 * Math.PI * i / n;
+ cosTable[i] = (float) Math.cos(angle);
+ sinTable[i] = (float) Math.sin(angle);
+ }
+ int depth = 0;
+ int maxRadix = 1;
+ for (int rest = n; rest > 1; depth++) {
+ int r = smallestFactor(rest);
+ maxRadix = Math.max(maxRadix, r);
+ rest /= r;
+ }
+ this.scratchRe = new float[depth + 1][n];
+ this.scratchIm = new float[depth + 1][n];
+ this.dftRe = new float[maxRadix];
+ this.dftIm = new float[maxRadix];
+ }
+
+ /** Transforms {@code re}/{@code im} of length {@code n} in place. */
+ void forward(float[] re, float[] im) {
+ transform(re, im, 0, 1, n, scratchRe[0], scratchIm[0], 0, 1);
+ System.arraycopy(scratchRe[0], 0, re, 0, n);
+ System.arraycopy(scratchIm[0], 0, im, 0, n);
+ }
+
+ /**
+ * Transforms the {@code len} samples starting at {@code inOff} with the given
+ * {@code stride}, writing the spectrum to {@code outRe}/{@code outIm} at
+ * {@code outOff}.
+ */
+ private void transform(float[] inRe, float[] inIm, int inOff, int stride, int len,
+ float[] outRe, float[] outIm, int outOff, int depth) {
+ if (len == 1) {
+ outRe[outOff] = inRe[inOff];
+ outIm[outOff] = inIm[inOff];
+ return;
+ }
+ int r = smallestFactor(len);
+ int m = len / r;
+
+ // Sub-transforms of the r interleaved subsequences, laid out end to end.
+ float[] subRe = scratchRe[depth];
+ float[] subIm = scratchIm[depth];
+ for (int j = 0; j < r; j++) {
+ transform(inRe, inIm, inOff + j * stride, stride * r, m,
+ subRe, subIm, outOff + j * m, depth + 1);
+ }
+
+ // Twiddle step for this transform size, expressed in top-level table units.
+ int unit = n / len;
+ for (int k2 = 0; k2 < m; k2++) {
+ for (int j = 0; j < r; j++) {
+ int idx = outOff + j * m + k2;
+ int t = (j * k2 * unit) % n;
+ float c = cosTable[t];
+ float s = sinTable[t];
+ dftRe[j] = subRe[idx] * c - subIm[idx] * s;
+ dftIm[j] = subRe[idx] * s + subIm[idx] * c;
+ }
+ combine(outRe, outIm, outOff + k2, m, r);
+ }
+ }
+
+ /** Length-{@code r} DFT of {@code dftRe}/{@code dftIm}, written with stride {@code m}. */
+ private void combine(float[] outRe, float[] outIm, int outOff, int m, int r) {
+ switch (r) {
+ case 2 -> {
+ outRe[outOff] = dftRe[0] + dftRe[1];
+ outIm[outOff] = dftIm[0] + dftIm[1];
+ outRe[outOff + m] = dftRe[0] - dftRe[1];
+ outIm[outOff + m] = dftIm[0] - dftIm[1];
+ }
+ default -> {
+ int unit = n / r;
+ for (int k1 = 0; k1 < r; k1++) {
+ float sumRe = 0;
+ float sumIm = 0;
+ for (int j = 0; j < r; j++) {
+ int t = (j * k1 % r) * unit;
+ float c = cosTable[t];
+ float s = sinTable[t];
+ sumRe += dftRe[j] * c - dftIm[j] * s;
+ sumIm += dftRe[j] * s + dftIm[j] * c;
+ }
+ outRe[outOff + k1 * m] = sumRe;
+ outIm[outOff + k1 * m] = sumIm;
+ }
+ }
+ }
+ }
+
+ private static int smallestFactor(int v) {
+ for (int f = 2; f * f <= v; f++) {
+ if (v % f == 0) return f;
+ }
+ return v;
+ }
+ }
+}
diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/vad/RnnNetwork.java b/ts3-client/core/src/main/java/com/ts3client/audio/vad/RnnNetwork.java
new file mode 100644
index 0000000..ea08670
--- /dev/null
+++ b/ts3-client/core/src/main/java/com/ts3client/audio/vad/RnnNetwork.java
@@ -0,0 +1,203 @@
+package com.ts3client.audio.vad;
+
+/**
+ * The {@code rnn_vad} network: a 42→24 dense layer, a 24-unit GRU and a 24→1
+ * dense output. Ported from WebRTC's {@code rnn_vad/rnn.cc}, {@code rnn_fc.cc} and
+ * {@code rnn_gru.cc}, using the activation approximations from RNNoise.
+ *
+ * The GRU carries state between frames, so the caller must feed frames in order and
+ * {@link #reset()} on a discontinuity. Silence resets the state, matching WebRTC.
+ */
+final class RnnNetwork {
+
+ private static final int INPUT_SIZE = 42;
+ private static final int FC1_SIZE = 24;
+ private static final int GRU_SIZE = 24;
+ private static final int NUM_GATES = 3; // update, reset, output
+
+ private final float[] fc1Weights; // [output][input]
+ private final float[] fc1Bias;
+ private final float[] fc2Weights;
+ private final float[] fc2Bias;
+
+ private final float[] gruWeights; // [gate][output][input]
+ private final float[] gruRecurrentWeights; // [gate][output][output]
+ private final float[] gruBias; // [gate][output]
+
+ private final float[] fc1Output = new float[FC1_SIZE];
+ private final float[] state = new float[GRU_SIZE];
+ private final float[] update = new float[GRU_SIZE];
+ private final float[] resetGate = new float[GRU_SIZE];
+ private final float[] resetTimesState = new float[GRU_SIZE];
+
+ RnnNetwork() {
+ fc1Weights = transposeAndScale(RnnVadWeights.INPUT_DENSE_WEIGHTS, INPUT_SIZE, FC1_SIZE);
+ fc1Bias = scale(RnnVadWeights.INPUT_DENSE_BIAS);
+ // The output layer is 24x1, so its stored layout already matches.
+ fc2Weights = scale(RnnVadWeights.OUTPUT_DENSE_WEIGHTS);
+ fc2Bias = scale(RnnVadWeights.OUTPUT_DENSE_BIAS);
+ gruWeights = transposeGruTensor(RnnVadWeights.HIDDEN_GRU_WEIGHTS, FC1_SIZE, GRU_SIZE);
+ gruRecurrentWeights =
+ transposeGruTensor(RnnVadWeights.HIDDEN_GRU_RECURRENT_WEIGHTS, GRU_SIZE, GRU_SIZE);
+ gruBias = transposeGruTensor(RnnVadWeights.HIDDEN_GRU_BIAS, 1, GRU_SIZE);
+ }
+
+ void reset() {
+ java.util.Arrays.fill(state, 0.0f);
+ }
+
+ /**
+ * Returns the speech probability for a feature vector. When {@code silence} is set the
+ * state is cleared and zero is returned without running the network.
+ */
+ float computeVadProbability(float[] featureVector, boolean silence) {
+ if (silence) {
+ reset();
+ return 0.0f;
+ }
+ for (int o = 0; o < FC1_SIZE; o++) {
+ float x = fc1Bias[o] + dot(featureVector, fc1Weights, o * INPUT_SIZE, INPUT_SIZE);
+ fc1Output[o] = tansigApproximated(x);
+ }
+ computeGruOutput(fc1Output);
+ float x = fc2Bias[0] + dot(state, fc2Weights, 0, GRU_SIZE);
+ return sigmoidApproximated(x);
+ }
+
+ private void computeGruOutput(float[] input) {
+ final int strideWeights = FC1_SIZE * GRU_SIZE;
+ final int strideRecurrent = GRU_SIZE * GRU_SIZE;
+
+ // u = sigmoid(W_u.i + R_u.s + b_u)
+ computeGate(input, 0, 0, 0, update);
+ // r = sigmoid(W_r.i + R_r.s + b_r)
+ computeGate(input, GRU_SIZE, strideWeights, strideRecurrent, resetGate);
+
+ // s' = u.s + (1 - u).ReLU(W_s.i + R_s.(s.r) + b_s)
+ for (int o = 0; o < GRU_SIZE; o++) {
+ resetTimesState[o] = state[o] * resetGate[o];
+ }
+ for (int o = 0; o < GRU_SIZE; o++) {
+ float x = gruBias[2 * GRU_SIZE + o];
+ x += dot(input, gruWeights, 2 * strideWeights + o * FC1_SIZE, FC1_SIZE);
+ x += dot(resetTimesState, gruRecurrentWeights,
+ 2 * strideRecurrent + o * GRU_SIZE, GRU_SIZE);
+ state[o] = update[o] * state[o] + (1.0f - update[o]) * Math.max(0.0f, x);
+ }
+ }
+
+ private void computeGate(float[] input, int biasOffset, int weightsOffset,
+ int recurrentOffset, float[] gate) {
+ for (int o = 0; o < GRU_SIZE; o++) {
+ float x = gruBias[biasOffset + o];
+ x += dot(input, gruWeights, weightsOffset + o * FC1_SIZE, FC1_SIZE);
+ x += dot(state, gruRecurrentWeights, recurrentOffset + o * GRU_SIZE, GRU_SIZE);
+ gate[o] = sigmoidApproximated(x);
+ }
+ }
+
+ private static float dot(float[] a, float[] weights, int weightsOffset, int len) {
+ float sum = 0;
+ for (int i = 0; i < len; i++) {
+ sum += a[i] * weights[weightsOffset + i];
+ }
+ return sum;
+ }
+
+ // ---- weight layout ----
+
+ private static float[] scale(byte[] src) {
+ float[] out = new float[src.length];
+ for (int i = 0; i < src.length; i++) {
+ out[i] = RnnVadWeights.SCALE * src[i];
+ }
+ return out;
+ }
+
+ /** Stored as {@code [input][output]}; the dot products want {@code [output][input]}. */
+ private static float[] transposeAndScale(byte[] src, int inputSize, int outputSize) {
+ float[] out = new float[src.length];
+ for (int o = 0; o < outputSize; o++) {
+ for (int i = 0; i < inputSize; i++) {
+ out[o * inputSize + i] = RnnVadWeights.SCALE * src[i * outputSize + o];
+ }
+ }
+ return out;
+ }
+
+ /**
+ * GRU tensors are stored as {@code [n][gate][output]} and are wanted as
+ * {@code [gate][output][n]}, where {@code n} is the input size (1 for the bias).
+ */
+ private static float[] transposeGruTensor(byte[] src, int n, int outputSize) {
+ float[] out = new float[src.length];
+ int strideSrc = NUM_GATES * outputSize;
+ int strideDst = n * outputSize;
+ for (int g = 0; g < NUM_GATES; g++) {
+ for (int o = 0; o < outputSize; o++) {
+ for (int i = 0; i < n; i++) {
+ out[g * strideDst + o * n + i] =
+ RnnVadWeights.SCALE * src[i * strideSrc + g * outputSize + o];
+ }
+ }
+ }
+ return out;
+ }
+
+ // ---- activations (RNNoise approximations, bit-compatible with WebRTC) ----
+
+ /** tanh sampled every 0.04, copied verbatim from RNNoise's {@code rnn_activations.h}. */
+ private static final float[] TANSIG_TABLE = {
+ 0.000000f, 0.039979f, 0.079830f, 0.119427f, 0.158649f, 0.197375f, 0.235496f, 0.272905f,
+ 0.309507f, 0.345214f, 0.379949f, 0.413644f, 0.446244f, 0.477700f, 0.507977f, 0.537050f,
+ 0.564900f, 0.591519f, 0.616909f, 0.641077f, 0.664037f, 0.685809f, 0.706419f, 0.725897f,
+ 0.744277f, 0.761594f, 0.777888f, 0.793199f, 0.807569f, 0.821040f, 0.833655f, 0.845456f,
+ 0.856485f, 0.866784f, 0.876393f, 0.885352f, 0.893698f, 0.901468f, 0.908698f, 0.915420f,
+ 0.921669f, 0.927473f, 0.932862f, 0.937863f, 0.942503f, 0.946806f, 0.950795f, 0.954492f,
+ 0.957917f, 0.961090f, 0.964028f, 0.966747f, 0.969265f, 0.971594f, 0.973749f, 0.975743f,
+ 0.977587f, 0.979293f, 0.980869f, 0.982327f, 0.983675f, 0.984921f, 0.986072f, 0.987136f,
+ 0.988119f, 0.989027f, 0.989867f, 0.990642f, 0.991359f, 0.992020f, 0.992631f, 0.993196f,
+ 0.993718f, 0.994199f, 0.994644f, 0.995055f, 0.995434f, 0.995784f, 0.996108f, 0.996407f,
+ 0.996682f, 0.996937f, 0.997172f, 0.997389f, 0.997590f, 0.997775f, 0.997946f, 0.998104f,
+ 0.998249f, 0.998384f, 0.998508f, 0.998623f, 0.998728f, 0.998826f, 0.998916f, 0.999000f,
+ 0.999076f, 0.999147f, 0.999213f, 0.999273f, 0.999329f, 0.999381f, 0.999428f, 0.999472f,
+ 0.999513f, 0.999550f, 0.999585f, 0.999617f, 0.999646f, 0.999673f, 0.999699f, 0.999722f,
+ 0.999743f, 0.999763f, 0.999781f, 0.999798f, 0.999813f, 0.999828f, 0.999841f, 0.999853f,
+ 0.999865f, 0.999875f, 0.999885f, 0.999893f, 0.999902f, 0.999909f, 0.999916f, 0.999923f,
+ 0.999929f, 0.999934f, 0.999939f, 0.999944f, 0.999948f, 0.999952f, 0.999956f, 0.999959f,
+ 0.999962f, 0.999965f, 0.999968f, 0.999970f, 0.999973f, 0.999975f, 0.999977f, 0.999978f,
+ 0.999980f, 0.999982f, 0.999983f, 0.999984f, 0.999986f, 0.999987f, 0.999988f, 0.999989f,
+ 0.999990f, 0.999990f, 0.999991f, 0.999992f, 0.999992f, 0.999993f, 0.999994f, 0.999994f,
+ 0.999994f, 0.999995f, 0.999995f, 0.999996f, 0.999996f, 0.999996f, 0.999997f, 0.999997f,
+ 0.999997f, 0.999997f, 0.999997f, 0.999998f, 0.999998f, 0.999998f, 0.999998f, 0.999998f,
+ 0.999998f, 0.999999f, 0.999999f, 0.999999f, 0.999999f, 0.999999f, 0.999999f, 0.999999f,
+ 0.999999f, 0.999999f, 0.999999f, 0.999999f, 0.999999f, 0.999999f, 1.000000f, 1.000000f,
+ 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
+ 1.000000f
+ };
+
+ private static float tansigApproximated(float x) {
+ // Tests are reversed to catch NaNs.
+ if (!(x < 8.0f)) {
+ return 1.0f;
+ }
+ if (!(x > -8.0f)) {
+ return -1.0f;
+ }
+ float sign = 1.0f;
+ if (x < 0.0f) {
+ x = -x;
+ sign = -1.0f;
+ }
+ int i = (int) Math.floor(0.5f + 25 * x);
+ float y = TANSIG_TABLE[i];
+ // Newton step back to x's own scale (undoing the table's factor of 25).
+ x -= 0.04f * i;
+ y = y + x * (1.0f - y * y) * (1.0f - y * x);
+ return sign * y;
+ }
+
+ private static float sigmoidApproximated(float x) {
+ return 0.5f + 0.5f * tansigApproximated(0.5f * x);
+ }
+}
diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/vad/RnnSpeechDetector.java b/ts3-client/core/src/main/java/com/ts3client/audio/vad/RnnSpeechDetector.java
new file mode 100644
index 0000000..4e94c62
--- /dev/null
+++ b/ts3-client/core/src/main/java/com/ts3client/audio/vad/RnnSpeechDetector.java
@@ -0,0 +1,140 @@
+package com.ts3client.audio.vad;
+
+import com.ts3client.audio.SpeechProbabilityDetector;
+
+/**
+ * {@link SpeechProbabilityDetector} backed by {@link RnnVad}, adapting an arbitrary
+ * capture rate and frame size to the detector's fixed 10 ms frames at 24 kHz.
+ *
+ * Capture frames rarely divide into 10 ms chunks exactly, so leftover samples are
+ * carried into the next call. The reported probability is that of the most recent complete
+ * 10 ms frame; a capture frame long enough to span several of them advances the
+ * detector several times, exactly as feeding it directly would.
+ */
+public final class RnnSpeechDetector implements SpeechProbabilityDetector {
+
+ /** Half-band FIR used when decimating; long enough to keep aliasing out of the passband. */
+ private static final int FILTER_TAPS = 32;
+
+ private final RnnVad vad = new RnnVad();
+
+ private final int sampleRate;
+ private final boolean decimateByTwo;
+ private final float[] filterCoeffs;
+ private final float[] filterState;
+ private int filterPos;
+
+ /** Resampled samples not yet consumed as a whole 10 ms frame. */
+ private final float[] pending = new float[RnnVad.FRAME_SIZE];
+ private int pendingCount;
+
+ /** Fractional read position, for capture rates that are not a multiple of 24 kHz. */
+ private double resamplePos;
+
+ public RnnSpeechDetector(int sampleRate) {
+ this.sampleRate = sampleRate;
+ this.decimateByTwo = sampleRate == 2 * RnnVad.SAMPLE_RATE;
+ if (decimateByTwo) {
+ this.filterCoeffs = lowPass(FILTER_TAPS, 0.25);
+ this.filterState = new float[FILTER_TAPS];
+ } else {
+ this.filterCoeffs = null;
+ this.filterState = null;
+ }
+ }
+
+ @Override
+ public double process(float[] frame, int length) {
+ if (sampleRate == RnnVad.SAMPLE_RATE) {
+ feed(frame, 0, length, 1);
+ } else if (decimateByTwo) {
+ decimateAndFeed(frame, length);
+ } else {
+ resampleAndFeed(frame, length);
+ }
+ return vad.probability();
+ }
+
+ @Override
+ public double probability() {
+ return vad.probability();
+ }
+
+ @Override
+ public void reset() {
+ vad.reset();
+ pendingCount = 0;
+ resamplePos = 0;
+ filterPos = 0;
+ if (filterState != null) {
+ java.util.Arrays.fill(filterState, 0.0f);
+ }
+ }
+
+ /** Appends samples (taking every {@code step}-th) and runs the detector per full frame. */
+ private void feed(float[] src, int offset, int length, int step) {
+ for (int i = offset; i < offset + length; i += step) {
+ push(src[i]);
+ }
+ }
+
+ /** 48 kHz (the common case): low-pass, then keep every other sample. */
+ private void decimateAndFeed(float[] frame, int length) {
+ for (int i = 0; i < length; i++) {
+ filterState[filterPos] = frame[i];
+ filterPos = (filterPos + 1) % FILTER_TAPS;
+ if ((i & 1) == 1) {
+ float acc = 0;
+ int idx = filterPos;
+ for (int t = 0; t < FILTER_TAPS; t++) {
+ acc += filterCoeffs[t] * filterState[idx];
+ idx = (idx + 1) % FILTER_TAPS;
+ }
+ push(acc);
+ }
+ }
+ }
+
+ /**
+ * Any other capture rate. Linear interpolation is enough here: the detector's own
+ * analysis is far coarser than the resampling error.
+ */
+ private void resampleAndFeed(float[] frame, int length) {
+ double ratio = (double) sampleRate / RnnVad.SAMPLE_RATE;
+ while (resamplePos < length) {
+ int i = (int) resamplePos;
+ double frac = resamplePos - i;
+ float a = frame[i];
+ float b = frame[Math.min(i + 1, length - 1)];
+ push((float) ((1 - frac) * a + frac * b));
+ resamplePos += ratio;
+ }
+ resamplePos -= length;
+ }
+
+ private void push(float sample) {
+ pending[pendingCount++] = sample;
+ if (pendingCount == RnnVad.FRAME_SIZE) {
+ vad.process(pending, 0);
+ pendingCount = 0;
+ }
+ }
+
+ /** Windowed-sinc low-pass with the given normalised cutoff, scaled to unit DC gain. */
+ private static float[] lowPass(int taps, double cutoff) {
+ float[] h = new float[taps];
+ double centre = (taps - 1) / 2.0;
+ double sum = 0;
+ for (int i = 0; i < taps; i++) {
+ double t = i - centre;
+ double sinc = (t == 0) ? 2 * cutoff : Math.sin(2 * Math.PI * cutoff * t) / (Math.PI * t);
+ double window = 0.54 - 0.46 * Math.cos(2 * Math.PI * i / (taps - 1)); // Hamming
+ h[i] = (float) (sinc * window);
+ sum += h[i];
+ }
+ for (int i = 0; i < taps; i++) {
+ h[i] /= (float) sum;
+ }
+ return h;
+ }
+}
diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/vad/RnnVad.java b/ts3-client/core/src/main/java/com/ts3client/audio/vad/RnnVad.java
new file mode 100644
index 0000000..69b54b3
--- /dev/null
+++ b/ts3-client/core/src/main/java/com/ts3client/audio/vad/RnnVad.java
@@ -0,0 +1,103 @@
+package com.ts3client.audio.vad;
+
+import static com.ts3client.audio.vad.VadConstants.BUF_SIZE_24K;
+import static com.ts3client.audio.vad.VadConstants.FEATURE_VECTOR_SIZE;
+import static com.ts3client.audio.vad.VadConstants.FRAME_10MS_24K;
+import static com.ts3client.audio.vad.VadConstants.FRAME_20MS_24K;
+import static com.ts3client.audio.vad.VadConstants.MAX_PITCH_24K;
+import static com.ts3client.audio.vad.VadConstants.NUM_LPC_COEFFICIENTS;
+
+/**
+ * WebRTC's {@code rnn_vad} speech-probability detector, ported to Java.
+ *
+ * This is the detector the TeamSpeak 3 client uses for its "Automatic" and "Hybrid"
+ * voice-activation modes: TS3 links the same network with byte-identical weights, so at a
+ * given threshold this gates on the same frames theirs does.
+ *
+ * Feed exactly {@link VadConstants#FRAME_10MS_24K} samples of 24 kHz mono audio in
+ * [-1, 1] per call, in order. Combines {@code features_extraction.cc} and {@code rnn.cc}.
+ */
+public final class RnnVad {
+
+ /** Samples per call: 10 ms at 24 kHz. */
+ public static final int FRAME_SIZE = FRAME_10MS_24K;
+
+ /** Sample rate this detector expects. */
+ public static final int SAMPLE_RATE = VadConstants.SAMPLE_RATE_24K;
+
+ /**
+ * WebRTC's audio processing carries float samples on the 16-bit integer scale, and the
+ * network's thresholds (the silence cut-off, the pitch energy floor) are calibrated for
+ * it. Callers pass [-1, 1], so scale on the way in.
+ */
+ private static final float INT16_SCALE = 32768.0f;
+
+ /**
+ * Rolling window of 24 kHz samples: one maximum pitch period of history followed by
+ * the most recent 20 ms.
+ */
+ private final float[] pitchBuffer = new float[BUF_SIZE_24K];
+ private final float[] lpResidual = new float[BUF_SIZE_24K];
+ private final float[] lpcCoeffs = new float[NUM_LPC_COEFFICIENTS];
+ private final float[] featureVector = new float[FEATURE_VECTOR_SIZE];
+
+ private final LinearPrediction linearPrediction = new LinearPrediction();
+ private final PitchEstimator pitchEstimator = new PitchEstimator();
+ private final SpectralFeatures spectralFeatures = new SpectralFeatures();
+ private final RnnNetwork network = new RnnNetwork();
+
+ private float probability;
+ private int pitchPeriod48k;
+
+ /** Clears all history. Call on a capture discontinuity. */
+ public void reset() {
+ java.util.Arrays.fill(pitchBuffer, 0.0f);
+ pitchEstimator.reset();
+ pitchPeriod48k = 0;
+ spectralFeatures.reset();
+ network.reset();
+ probability = 0.0f;
+ }
+
+ /** Speech probability in [0, 1] from the most recent {@link #process} call. */
+ public float probability() {
+ return probability;
+ }
+
+ /**
+ * Pitch period of the most recent frame, expressed at 48 kHz as the network's
+ * feature is. Exposed for diagnostics and tests.
+ */
+ public int pitchPeriod48kHz() {
+ return pitchPeriod48k;
+ }
+
+ /**
+ * Analyses one 10 ms frame and returns the speech probability in [0, 1].
+ *
+ * @param frame mono samples in [-1, 1]
+ * @param offset index of the first sample
+ */
+ public float process(float[] frame, int offset) {
+ System.arraycopy(pitchBuffer, FRAME_10MS_24K, pitchBuffer, 0, BUF_SIZE_24K - FRAME_10MS_24K);
+ int start = BUF_SIZE_24K - FRAME_10MS_24K;
+ for (int i = 0; i < FRAME_10MS_24K; i++) {
+ pitchBuffer[start + i] = frame[offset + i] * INT16_SCALE;
+ }
+
+ // Search the pitch on the whitened signal so formants don't dominate the peak.
+ linearPrediction.computeLpcCoefficients(pitchBuffer, lpcCoeffs);
+ LinearPrediction.computeLpResidual(lpcCoeffs, pitchBuffer, lpResidual);
+ pitchPeriod48k = pitchEstimator.estimate(lpResidual);
+ // Normalisation based on the training-set statistics.
+ featureVector[FEATURE_VECTOR_SIZE - 2] = 0.01f * (pitchPeriod48k - 300);
+
+ // Compare the most recent 20 ms against the 20 ms one pitch period earlier.
+ int laggedOffset = MAX_PITCH_24K - pitchPeriod48k / 2;
+ boolean silence = spectralFeatures.checkSilenceComputeFeatures(
+ pitchBuffer, BUF_SIZE_24K - FRAME_20MS_24K, pitchBuffer, laggedOffset, featureVector);
+
+ probability = network.computeVadProbability(featureVector, silence);
+ return probability;
+ }
+}
diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/vad/RnnVadWeights.java b/ts3-client/core/src/main/java/com/ts3client/audio/vad/RnnVadWeights.java
new file mode 100644
index 0000000..836d0cc
--- /dev/null
+++ b/ts3-client/core/src/main/java/com/ts3client/audio/vad/RnnVadWeights.java
@@ -0,0 +1,273 @@
+package com.ts3client.audio.vad;
+
+/**
+ * Quantised weights of WebRTC's {@code rnn_vad} network, copied verbatim from
+ * {@code third_party/rnnoise/src/rnn_vad_weights.cc} (BSD, Xiph/Jean-Marc Valin).
+ *
+ * These are the same 4585 bytes the TeamSpeak 3 client links in, so our detector
+ * produces the same speech probabilities theirs does.
+ *
+ * Values are stored quantised; {@link #SCALE} converts them to float.
+ */
+final class RnnVadWeights {
+
+ /** Dequantisation factor applied to every table below. */
+ static final float SCALE = 1.0f / 256.0f;
+
+ /** FC1 weights, laid out as [input][output] (42 x 24). */
+ static final byte[] INPUT_DENSE_WEIGHTS = {
+ -10, 0, -3, 1, -8, -6, 3, -13, 1, 0, -3, -7, -5, -3, 6, -1, -6, 0, -6, -4, -1, -2, 1, 1, -7, 2,
+ 21, 10, -5, -20, 24, 23, 37, 8, -2, 33, -6, 22, 13, -2, 50, 8, 13, 1, -15, 30, -10, 30, 0, 3, 5,
+ 27, 1, 4, -3, 41, 56, 35, -2, 49, -13, 11, 13, -2, -47, 5, -16, -60, -15, 77, -17, 26, -3, 14,
+ -21, 19, -5, -19, -13, 0, 10, 14, 9, 31, -13, -41, -10, 4, 22, 18, -48, -6, -10, 62, -3, -18,
+ -14, 12, 26, -28, 3, 14, 25, -13, -19, 6, 5, 36, -3, -65, -12, 0, 31, -7, -9, 101, -4, 26, 16,
+ 17, -12, -12, 14, -36, -3, 5, -15, 21, 2, 30, -3, 38, -4, 1, -6, 7, -7, 14, 38, -22, -30, -3,
+ -7, 3, -39, -70, -126, 25, 34, 94, -67, -22, -33, 83, -47, -118, 4, 70, 33, 25, 62, -128, -76,
+ -118, -113, 49, -12, -100, -18, -114, -33, 43, 32, 61, 40, -9, -106, 2, 36, -100, -40, -5, 20,
+ -75, 61, -51, -9, 126, -27, -52, 5, -24, -21, -126, -114, -12, 15, 106, -2, 73, -125, 50, 13,
+ -120, 35, 35, 4, -61, 29, -124, 6, -53, -69, -125, 64, -89, 36, -107, -103, -7, 27, 121, 69, 77,
+ -35, 35, 95, -125, -49, 97, -45, -43, -23, 23, -28, -65, -118, 2, 8, -126, 27, -97, 92, 5, 55,
+ 82, 17, -57, -115, 37, 8, -106, -46, 41, -2, 21, -44, 8, -73, -58, -39, 34, 89, -95, 95, -117,
+ 120, -58, 31, 123, 1, -32, -109, -110, 60, -120, -43, -74, 5, 91, 26, 21, 114, 82, -83, -126,
+ 123, 22, -16, -67, 25, -83, 46, 48, -34, -121, -124, -63, -35, -9, 31, 82, 123, 6, -3, 117, 93,
+ -2, -13, -36, 124, -112, -6, -102, -5, -33, -15, 44, -69, -127, -23, -40, -34, -85, 68, 83, -1,
+ 40, 8, 84, 118, -58, -55, -102, 123, -55, -14, -123, 44, -63, -14, 21, 35, 16, 24, -126, -13,
+ -114, 35, 20, -36, 61, -9, 97, 34, 19, -32, -109, 76, -104, 99, -119, 45, -125, -51, -28, -8,
+ -69, -8, 125, -45, -93, 113, 103, -41, -82, 52, 7, 126, 0, -40, 104, 55, -58, 17, -124, -93,
+ -58, 8, -45, 1, 56, -123, 108, -47, -23, 115, 127, 17, -68, -13, 116, -82, -44, 45, 67, -120,
+ -101, -15, -125, 120, -113, 17, -48, -73, 126, -64, -86, -118, -19, 112, -1, -66, -27, -62, 121,
+ -86, -58, 50, 89, -38, -75, 95, -111, 12, -113, 2, -68, 2, -94, -121, 91, -5, 0, 79, 43, -7,
+ -18, 79, 35, -38, 47, 1, -45, 83, -50, 102, 32, 55, -96, 15, -122, -69, 45, -27, 91, -62, -30,
+ 46, -95, 22, -72, -97, -1, 14, -122, 28, 127, 61, -126, 121, 9, 68, -120, 49, -60, 90, 3, 43,
+ 68, 54, 34, -10, 28, 21, -24, -54, 22, -113, -12, 82, -2, -17, -9, 127, 8, 116, -92, 0, -70,
+ -33, 123, 66, 116, -74, -4, 74, -72, -22, -47, 1, -83, -60, -124, 1, 122, -57, -43, 49, 40,
+ -126, -128, -8, -29, 28, -24, -123, -121, -70, -93, -37, -126, 11, -125, -37, 11, -31, -51,
+ -124, 116, -128, 8, -25, 109, 75, -12, 7, 8, 10, 117, 124, -128, -128, 29, -26, 101, 21, -128,
+ 87, 8, -39, 23, -128, 127, -127, 74, -55, 74, 112, 127, 4, 55, 44, -92, 123, 34, -93, 47, -21,
+ -92, 17, 49, -121, 92, 7, -126, -125, 124, -74, 3, -59, 18, -91, 3, -9, 9, 56, 116, 7, -29, 33,
+ 87, -21, -128, -13, 57, 74, 9, -29, -61, -97, -21, -95, -12, -114, 16, 82, 125, -7, 10, -24, 9,
+ 77, -128, -102, -25, 3, -126, 10, 13, -18, 51, 26, 127, -79, 35, 51, 12, -50, -24, 1, -7, 22,
+ 81, 65, 120, -30, -38, 85, 122, -4, -106, -11, 27, 53, 41, 8, -104, -66, -38, -124, 10, 12, 76,
+ 117, -109, 9, 11, 2, -18, 3, 113, -16, -79, -39, -123, -20, -128, 2, 13, -33, -58, 10, 84, -104,
+ 13, 64, 109, 1, 54, -12, 28, 24, 63, -126, 118, -82, 46, -12, -15, 14, -43, 60, 22, -32, -19,
+ -46, 91, -107, 24, -94, 26, -47, 125, 6, 58, -15, -75, -26, -38, -35, 103, -16, -17, -13, 63,
+ -2, 45, -45, -73, -23, 70, -87, 51, -17, 53, 76, 14, -18, -31, -14, 103, 8, 21, -28, -33, -20,
+ -47, 6, 39, 40, -30, 7, -76, 55, 31, -20, -21, -59, 1, 25, -11, 17, 5, -13, -39, 0, -76, 50,
+ -33, -29, -50, -16, -11, -12, -1, -46, 40, -10, 65, -19, 21, -41, -32, -83, -19, -4, 49, -60,
+ 118, -24, -46, 9, 102, -20, 8, -19, 25, 31, -3, -37, 0, 25, 7, 29, 2, -39, 127, -64, -20, 64,
+ 115, -30, 36, 100, 35, 122, 127, 127, -127, 127, -127, 19, 127, -89, -79, -32, 39, -127, 125,
+ -80, 126, -127, 26, 8, 98, -8, -57, -90, -50, 126, 61, 127, -126, 40, -106, -68, 104, -125,
+ -119, 11, 10, -127, 66, -56, -12, -126, -104, 27, 75, 38, -124, -126, -125, 84, -123, -45, -114,
+ -128, 127, 103, -101, -124, 127, -11, -23, -123, 92, -123, 24, 126, 41, -2, -39, -27, -94, 40,
+ -112, -48, 127, 58, 14, 38, -75, -64, 73, 117, 100, -119, -11, 6, 32, -126, -14, 35, 121, -10,
+ 54, -60, 89, -3, 69, -25, -20, 43, -86, -34, 24, 27, 7, -81, -99, -23, -16, -26, 13, 35, -97,
+ 80, -29, -13, -121, -12, -65, -94, 70, -89, -126, -95, 88, 33, 96, 29, -90, 69, 114, -78, 65,
+ 90, -47, -47, 89, 1, -12, 3, 8, 30, 5, 2, -30, -1, 6, -7, 10, -4, 46, -27, -40, 22, -6, -17, 45,
+ 24, -9, 23, -14, -63, -26, -12, -57, 27, 25, 55, -76, -47, 21, 34, 33, 26, 17, 14, 6, 9, 26, 25,
+ -25, -25, -18
+ };
+
+ /** FC1 bias (24). */
+ static final byte[] INPUT_DENSE_BIAS = {
+ 38, -6, 127, 127, 127, -43, -127, 78, 127, 5, 127, 123, 127, 127, -128, -76, -126, 28, 127, 125,
+ -30, 127, -89, -20
+ };
+
+ /** GRU1 input weights, [input][gate][output] (24 x 3 x 24). */
+ static final byte[] HIDDEN_GRU_WEIGHTS = {
+ -124, 23, -123, -33, -95, -4, 8, -84, 4, 101, -119, 116, -4, 123, 103, -51, 29, -124, -114, -49,
+ 31, 9, 75, -128, 0, -49, 37, -50, 46, -21, -63, -104, 54, 82, 33, 21, 70, 127, -9, -79, -39,
+ -23, -127, 107, 122, -96, -46, -18, -39, 13, -28, -48, 14, 56, -52, 49, -1, -121, 25, -18, -36,
+ -52, -57, -30, 54, -124, -26, -47, 10, 39, 12, 2, 9, -127, -128, 102, 21, 11, -64, -71, 89,
+ -113, -111, 54, 31, 94, 121, -40, 30, 40, -109, 73, -9, 108, -92, 2, -127, 116, 127, 127, -122,
+ 95, 127, -37, -127, 28, 89, 10, 24, -104, -62, -67, -14, 38, 14, -71, 22, -41, 20, -50, 39, 63,
+ 86, 127, -18, 79, 4, -51, 2, 33, 117, -113, -78, 56, -91, 37, 34, -45, -44, -22, 21, -16, 56,
+ 30, -84, -79, 38, -74, 127, 9, -25, 2, 82, 61, 25, -26, 26, 11, 117, -65, 12, -58, 42, -62, -93,
+ 11, 11, 124, -123, 80, -125, 11, -90, 42, 94, 4, -109, -1, 85, -52, 45, -26, -27, 77, -5, 30,
+ 90, 0, 95, -7, 53, 29, -82, 22, -9, 74, 2, -12, -73, 114, 97, -64, 122, -77, 43, 91, 86, 126,
+ 106, 72, 90, -43, 46, 96, -51, 21, 22, 68, 22, 41, 79, 75, -46, -105, 23, -116, 127, -123, 102,
+ 57, 85, 10, -29, 34, 125, 126, 124, 81, -15, 54, 96, -128, 39, -124, 103, 74, 126, 127, -50,
+ -71, -122, -64, 93, -75, 71, 105, 122, 123, 126, 122, -127, 33, -63, -74, 124, -71, 33, 41, -56,
+ 19, 6, 65, 41, 90, -116, -3, -46, 75, -13, 98, -74, -42, 74, -95, -96, 81, 24, 32, -19, -123,
+ 74, 55, 109, 115, 0, 32, 33, 12, -20, 9, 127, 127, -61, 79, -48, -54, -49, 101, -9, 27, -106,
+ 74, 119, 77, 87, -126, -24, 127, 124, 31, 34, 127, 40, 3, -90, 127, 23, 57, -53, 127, -69, -88,
+ -33, 127, 19, -46, -9, -125, 13, -126, -113, 127, -41, 46, 106, -62, 3, -10, 111, 49, -34, -24,
+ -20, -112, 11, 101, -50, -34, 50, 65, -64, -106, 70, -48, 60, 9, -122, -45, 15, -112, -26, -4,
+ 1, 39, 23, 58, -45, -80, 127, 82, 58, 30, -94, -119, 51, -89, 95, -107, 30, 127, 125, 58, -52,
+ -42, -38, -20, -122, 115, 39, -26, 5, 73, 13, -39, 43, -23, -20, -125, 23, 35, 53, -61, -66, 72,
+ -20, 33, 8, 35, 4, 7, 18, 19, 16, -45, -50, -71, 31, -29, -41, -27, 10, 14, 27, 9, -23, 98, 6,
+ -94, 92, 127, -114, 59, -26, -100, -62, -127, -17, -85, -60, 126, -42, -6, 33, -120, -26, -126,
+ -127, -35, -114, -31, 25, -126, -100, -126, -64, -46, -31, 30, 25, -74, -111, -97, -81, -104,
+ -114, -19, -9, -116, -69, 22, 30, 59, 8, -51, 16, -97, 18, -4, -89, 80, -50, 3, 36, -67, 56, 69,
+ -26, 107, -10, 58, -28, -4, -57, -72, -111, 0, -75, -119, 14, -75, -49, -66, -49, 8, -121, 22,
+ -54, 121, 30, 54, -26, -126, -123, 56, 5, 48, 21, -127, -11, 23, 25, -82, 6, -25, 119, 78, 4,
+ -104, 27, 61, -48, 37, -13, -52, 50, -50, 44, -1, -22, -43, -59, -78, -67, -32, -26, 9, -3, 40,
+ 16, 19, 3, -9, 20, -6, -37, 28, 39, 17, -19, -10, 1, 6, -59, 74, 47, 3, -119, 0, -128, -107,
+ -25, -22, -69, -23, -111, -42, -93, -120, 90, -85, -54, -118, 76, -79, 124, 101, -77, -75, -17,
+ -71, -114, 68, 55, 79, -1, -123, -20, 127, -65, -123, -128, -87, 123, 9, -115, -14, 7, -4, 127,
+ -79, -115, 125, -28, 89, -83, 49, 89, 119, -69, -5, 12, -49, 60, 57, -24, -99, -110, 76, -83,
+ 125, 73, 81, 11, 8, -45, 1, 83, 13, -70, -2, 97, 112, -97, 53, -9, -94, 124, 44, -49, -24, 52,
+ 76, -110, -70, -114, -12, 72, -4, -114, 43, -43, 81, 102, -84, -27, 62, -40, 52, 58, 124, -35,
+ -51, -123, -43, 56, -75, -34, -35, -106, 93, -43, 14, -16, 46, 62, -97, 21, 30, -53, 21, -11,
+ -33, -20, -95, 4, -126, 12, 45, 20, 108, 85, 11, 20, -40, 99, 4, -25, -18, -23, -12, -126, -55,
+ -20, -44, -51, 91, -127, 127, -44, 7, 127, 78, 38, 125, -6, -94, -103, 73, 126, -126, 18, 59,
+ -46, 106, 76, 116, -31, 75, -4, 92, 102, 32, -31, 73, 42, -21, -28, 57, 127, -8, -107, 115, 124,
+ -94, -4, -128, 29, -57, 70, -82, 50, -13, -44, 38, 67, -93, 6, -39, -46, 56, 68, 27, 61, 26, 18,
+ -72, 127, 22, 18, -31, 127, 61, -65, -38, 1, -67, -1, 8, -73, 46, -116, -94, 58, -49, 71, -40,
+ -63, -82, -20, -60, 93, 76, 69, -106, 34, -31, 4, -25, 107, -18, 45, 4, -61, 126, 54, -126,
+ -125, 41, 19, 44, 32, -98, 125, -24, 125, -96, -125, 15, 87, -4, -90, 18, -40, 28, -69, 67, 22,
+ 41, 39, 7, -48, -44, 12, 69, -13, 2, 44, -38, 111, -7, -126, -22, -9, 74, -128, -36, -7, -123,
+ -15, -79, -91, -37, -127, -122, 104, 30, 7, 98, -37, 111, -116, -47, 127, -45, 118, -111, -123,
+ -120, -77, -64, -125, 124, 77, 111, 77, 18, -113, 117, -9, 67, -77, 126, 49, -20, -124, 39, 41,
+ -124, -34, 114, -87, -126, 98, -20, 59, -17, -24, 125, 107, 54, 35, 33, -44, 12, -29, 125, -71,
+ -28, -63, -114, 28, -17, 121, -36, 127, 89, -122, -49, -18, -48, 17, 24, 19, -64, -128, 13, 86,
+ 45, 13, -49, 55, 84, 48, 80, -39, 99, -127, 70, -33, 30, 50, 126, -65, -117, -13, -20, -24, 127,
+ 115, -72, -104, 63, 126, -42, 57, 17, 46, 21, 119, 110, -100, -60, -112, 62, -33, 28, 26, -22,
+ -60, -33, -54, 78, 25, 32, -114, 86, 44, 26, 43, 76, 121, 19, 97, -2, -3, -73, -68, 6, -116, 6,
+ -43, -97, 46, -128, -120, -31, -119, -29, 16, 16, -126, -128, -126, -46, -9, -3, 92, -31, -76,
+ -126, -3, -107, -12, -23, -69, 5, 51, 27, -42, 23, -70, -128, -29, 22, 29, -126, -55, 50, -71,
+ -3, 127, 44, -27, -70, -63, -66, -70, 104, 86, 115, 29, -92, 41, -90, 44, -11, -28, 20, -11,
+ -63, -16, 43, 31, 17, -73, -31, -1, -17, -11, -39, 56, 18, 124, 72, -14, 28, 69, -121, -125, 34,
+ 127, 63, 86, -80, -126, -125, -124, -47, 124, 77, 124, -19, 23, -7, -50, 96, -128, -93, 102,
+ -53, -36, -87, 119, -125, 92, -126, 118, 102, 72, -2, 125, 10, 97, 124, -125, 125, 71, -20, -47,
+ -116, -121, -4, -9, -32, 79, -124, -36, 33, -128, -74, 125, 23, 127, -29, -115, -32, 124, -89,
+ 32, -107, 43, -17, 24, 24, 18, 29, -13, -15, -36, 62, -91, 4, -41, 95, 28, -23, 6, 46, 84, 66,
+ 77, 68, -70, -1, -23, -6, 65, 70, -21, 9, 77, -12, 2, -118, 4, 9, -108, 84, 52, 2, 52, 13, -10,
+ 58, -110, 18, 66, -95, -23, 70, 31, -3, 56, 56, -3, -7, 1, -27, -48, -61, 41, -4, 10, -62, 32,
+ -7, -24, 9, -48, -60, -4, 79, -20, -38, -76, 68, -49, -97, 0, -15, 5, -100, -49, -95, -99, -115,
+ -9, -40, 10, 104, 13, 56, 127, -27, -109, -94, -118, -102, -44, -85, 52, 127, -4, 14, 62, 121,
+ -122, -26, -79, -42, -34, 1, 25, -38, -79, -58, -31, -31, -90, -30, -123, 32, -56, 125, 66, 124,
+ -1, 3, 91, -103, -7, 23, 78, -18, 9, 69, -69, 76, -38, -33, -2, -98, 18, 106, 84, 55, 87, -47,
+ 35, -124, 64, 41, -14, 46, 25, -2, 120, -21, 82, 19, -79, -37, -3, -8, -16, 21, 19, -5, -28,
+ -112, 39, -6, -30, 53, -69, 53, 46, 127, 123, 78, 20, 28, -7, 73, 72, 17, -40, 41, 111, 57, 32,
+ -95, 29, 28, -39, -65, 54, -20, -63, 29, -67, 3, 44, -57, -47, 11, 61, -22, -44, 61, 48, -100,
+ 20, 125, 96, -24, -16, 3, -69, -126, 74, -125, 9, 45, -67, -123, -59, -72, 118, 69, 45, 50, -57,
+ 67, 13, -66, -106, 47, 62, 22, -1, -22, -25, -40, -125, 3, 125, 32, 102, -56, -25, -75, -30,
+ 122, 60, -13, 36, -73, 7, -84, 124, 40, -118, 17, -87, -118, -8, 3, -27, 111, -40, 40, -51, 127,
+ 125, -45, -30, -54, 46, 80, -1, -30, 101, -17, 18, 26, 54, 7, -12, 1, -127, 123, -122, -27, -75,
+ 64, 10, 25, -15, -44, 127, -127, 5, -84, -81, -7, 19, -26, 126, 15, 116, -126, 14, -76, 44, 62,
+ -110, -124, 125, -29, -87, -3, -69, 82, 90, 57, -123, 123, 100, -19, -51, -32, 69, 37, -57,
+ -128, -124, -72, -13, 51, -7, -45, -73, 5, 99, -26, -117, -96, -109, 4, -31, -12, 0, 31, -42,
+ -27, 12, -81, 118, 39, 83, 14, 41, -126, 107, -82, 94, -116, -122, -47, -109, -84, -128, -35,
+ -56, 66, 8, -65, 19, 42, -46, -72, -109, 41, 43, -127, -113, 58, 127, 42, -75, -1, 65, 117, -55,
+ -113, -123, 124, 43, -96, -115, -19, 68, 15, 94, 3, 75, 0, 34, 9, 42, 110, -48, 92, -76, 99,
+ -17, 27, 32, 13, 125, 50, -17, 56, 4, 53, 34, -8, 99, 80, -126, -21, -65, -11, -46, 44, -81, -3,
+ -121, 123, 66, -81, -84, 119, 127, 84, 105, 45, -66, -42, -23, 32, -25, 12, 111, 127, 88, 125,
+ 30, 24, -127, -9, -54, 127, -116, -119, 88, 70, 94, -120, 35, -93, 15, 22, -21, 25, -110, -123,
+ -45, 8, -109, 125, -122, -86, -126, 8, -14, -120, -45, -45, 69, -125, -122, 6, 81, 86, 125, 95,
+ 54, 77, 54, -123, 126, -85, -117, 56, 11, 0, -61, -91, -12, -2, -113, -3, -15, -122, -63, -91,
+ 10, 84, -111, 125, 93, 21, 62, -78, -116, 13, -57, 28, -124, 126, 110, 12, 15, 95, 15, -19,
+ -125, -97, 52, -7, 101, 9, 20, -125, -26, -56, 72, 77, 12, -126, 22, -29, 47, 62, 95, 112, 69,
+ 32, 97, -83, -8, -5, 67, -63, -123, 79, 59, 0, -6, -17, 4, -111, -52, 27, 65, 0
+ };
+
+ /** GRU1 recurrent weights, same layout (24 x 3 x 24). */
+ static final byte[] HIDDEN_GRU_RECURRENT_WEIGHTS = {
+ 65, 83, 35, 56, 24, -34, -28, -2, 125, 19, 42, -9, 124, -53, 24, -87, 11, 35, -81, -35, -125,
+ -31, 123, -21, 33, -91, 113, -93, 45, -6, 53, 38, -92, 8, -27, 87, 4, 43, 43, 10, -128, -128,
+ -46, 127, -38, -45, 25, -87, 19, 5, 52, -96, -23, -29, 121, -126, -24, -20, -2, 69, -50, 6, 71,
+ -81, -125, 90, -94, 1, -38, 36, 89, 17, -60, 71, -48, 18, -15, 44, -18, 59, 11, 114, -51, 32,
+ 110, 1, 4, 109, -24, 127, 27, 60, 88, 24, 45, -59, 75, -36, 8, 57, -32, -25, 13, 126, -89, -61,
+ -76, 127, 18, -62, -68, 23, -113, 5, 126, 43, -88, 26, -78, 18, 75, 21, 9, -74, 20, 41, 126,
+ -118, -15, 9, 116, 126, -127, 34, -6, 126, -128, -53, -54, -55, -121, 70, 127, -12, -68, 82,
+ -25, 104, -126, 126, -21, -26, 124, -75, -127, -120, 13, 61, -64, -108, -63, -65, -44, -35, -61,
+ -39, 109, -74, 113, -3, 108, -30, 125, 120, 39, 125, -128, -95, -99, 111, 9, 25, 114, -75, -92,
+ -54, -12, -32, -38, 10, 31, 10, 63, 51, 40, -99, 74, 4, 50, -128, -36, -35, -11, -28, -126, -7,
+ 66, -58, -126, -22, -83, -61, -127, 49, 126, -8, 7, 62, 36, -11, -32, -44, 63, 116, 41, 65,
+ -127, 126, 63, -30, -96, 74, -92, 127, 38, -18, -128, 68, -5, 101, -4, 85, 58, 79, 0, -58, 8,
+ 119, -70, -1, -79, -68, 114, -28, -90, -6, -112, 2, 127, -8, 10, 55, -59, -126, 127, 125, 80,
+ 72, 35, -54, 95, -124, -124, 79, 23, -46, -61, -127, -100, 99, -77, 8, -87, 5, -2, 49, 85, 7,
+ -71, 82, 53, -41, 22, -22, -93, -103, 6, 52, -56, 14, -8, -111, 85, 16, 54, 32, -118, -24, 61,
+ -53, 96, -70, -5, -17, -67, -84, -7, -82, -107, -96, 21, -83, -58, 50, 12, -126, -1, -28, 34,
+ -126, 115, 17, 91, 1, -127, 72, 11, 126, -81, 6, 96, -8, 77, 15, -6, 63, -27, 20, -123, -109,
+ 85, -79, -17, 126, -92, 2, -61, 20, 14, 17, 121, 123, 30, 57, 120, 127, 57, 42, 117, 98, 67, 39,
+ -20, -70, 100, 7, 125, 122, 40, 16, -79, 125, 83, 41, -106, -57, 24, 55, 27, -66, -111, -44, -7,
+ -43, -66, 121, 42, -128, -45, 35, 15, -127, 34, -35, -34, -40, -18, -6, 63, 111, 31, 116, 127,
+ 19, 24, -71, -39, 34, 11, 19, -40, 27, 12, 106, -10, 56, -82, -106, -2, -50, -52, 114, -126,
+ -34, -43, -68, 10, 76, 57, -118, -128, 37, -104, 76, 125, 3, -76, 127, -29, 84, -94, -15, 55,
+ 125, 79, 127, -57, -125, 104, -68, 126, 126, -77, 51, 45, 33, -109, 115, -11, 1, 95, -121, -5,
+ -9, -126, -114, 39, 68, -126, -107, -51, -42, 24, -8, 51, -27, -43, 66, -45, 62, -98, -109, 69,
+ 67, 0, -125, -128, 49, 31, 126, -122, 2, -55, -67, -126, -70, -128, -125, -77, 25, 16, -8, -102,
+ 11, -75, 82, 38, -5, 5, 19, 34, 47, -127, -93, 21, 24, -97, -18, 31, 39, 34, -20, 22, 123, 7,
+ -77, -81, -46, -9, 1, 23, 39, -127, -43, -8, -50, 10, -21, 59, -9, -4, -13, -27, 44, 127, 52,
+ -47, 70, -43, 52, 101, -49, 27, 45, 49, 33, -125, 55, 114, 20, -1, 76, -24, -96, 105, 24, 126,
+ 75, -21, -105, 13, -42, 40, 126, -30, -39, -95, 125, -63, 11, 6, 125, 125, -14, 5, 42, -61, -4,
+ 49, 88, 6, -107, -28, 19, -29, 47, 126, 6, -46, -89, -18, 91, -20, -6, 118, -21, -22, 39, 115,
+ 11, -42, 54, 73, -55, -77, 62, -27, -59, -99, -12, -127, -40, 56, -3, -124, -91, 71, -111, 6,
+ -19, 82, -24, -35, 102, -42, 7, -126, -126, -125, 18, 98, -52, 127, 105, -52, 40, -83, 126,
+ -122, 109, 5, 127, 48, 6, 5, -125, 100, -16, 29, 85, -89, 8, 4, 41, 62, -127, 62, 122, 85, 122,
+ -107, 8, -125, 93, -127, 127, 102, 19, 19, -66, 41, -42, 114, 127, -48, -117, -29, -6, -73,
+ -102, -3, -19, 0, 88, 42, 87, -117, -20, 2, 122, 28, 63, 71, 66, 120, 93, 124, -43, 49, 103, 31,
+ 90, -91, -22, -126, 26, -24, -21, 51, -126, 87, -103, -69, -10, -66, -23, 20, 97, 36, 25, -127,
+ 30, -20, -63, 30, 51, -116, 23, 40, -39, 36, -83, -77, -25, -50, 110, 14, 13, -109, 125, -65,
+ -55, -87, 124, -126, -32, -72, -108, 127, 127, -125, -124, 61, 121, 102, -128, -127, 16, 100,
+ 127, -124, -68, 72, -93, -128, 43, -93, -19, -125, -97, -113, -33, 83, 127, -44, 127, -75, 127,
+ 16, 44, 50, -122, 23, 118, 46, 19, 26, -128, 10, 4, 99, -14, -82, -13, 30, 125, 57, 65, 60, -71,
+ 35, 98, 28, 7, 1, 43, 89, 70, 75, 121, -59, 82, -126, -53, -16, -116, -65, 52, -52, 0, 80, 35,
+ 45, -61, 46, 8, 107, 27, -26, -118, 90, 57, -10, 7, -15, 0, -39, -4, 12, 29, -1, 116, 84, 79,
+ 119, 125, -59, 28, -6, -25, -43, 2, 90, 79, 67, 103, -82, 2, -6, 125, 19, 73, 0, -105, 112, -17,
+ 104, 107, 124, 106, 19, 56, -44, 55, -112, 6, -39, -83, 126, -93, -98, 57, -120, -23, -38, 2,
+ -31, -48, 106, 127, 127, 69, 16, 110, 71, 104, 62, -12, -22, 42, -37, -94, 34, -1, -32, -12,
+ -124, -47, -13, 60, -75, -66, 58, -127, -2, 64, 76, -106, 73, -49, -31, 127, 126, 31, 16, 127,
+ -110, 107, -16, -53, 20, 69, -14, -125, 59, -44, 15, 120, 125, 125, 43, 6, 19, -58, 127, 127,
+ 43, 16, 82, 97, -127, 127, -93, -41, 88, 0, 77, -15, 116, 16, -124, -31, -3, 95, -40, -126, -54,
+ -126, -83, -8, -59, 6, 67, -29, 4, 124, -10, 112, -28, -8, 85, -21, 45, 84, 6, -8, 11, 72, 32,
+ 84, -62, 77, 2, -36, 75, 31, -50, 116, 126, 119, -88, -55, -14, -37, 126, 40, -108, -6, -6, 57,
+ 64, -28, -76, 30, -117, -93, 31, -92, -44, -64, 94, 58, 65, 114, 41, 47, 71, 42, -26, 99, -126,
+ 57, -5, 74, -19, -113, -1, 67, -21, 126, 1, -3, 33, 60, -82, 37, -48, 89, 114, -38, 127, -114,
+ 35, 58, -5, 21, -46, 121, -123, -43, 127, 115, 123, 122, -101, 126, 127, 81, 52, 89, -127, 102,
+ 42, 117, -9, -2, 125, 127, 110, 96, 120, 66, 70, 124, 55, 84, -38, -58, 119, -127, -16, -79,
+ 123, 18, -127, -50, -38, 120, -85, 1, 7, -56, 108, -77, -2, 21, 37, 1, 13, -105, -69, 28, -87,
+ 33, -104, -51, 126, 41, 3, -121, 28, 71, 58, 86, -8, 127, 94, -55, 125, 40, -19, 127, -33, -87,
+ -23, 7, -111, -68, 9, 84, -119, 55, -82, 78, -37, -20, -9, -23, 53, -13, 15, -46, 116, 126,
+ -127, 56, -126, 125, -7, -1, 45, 26, 125, 121, 29, 47, -86, 30, 10, 76, -125, -7, 23, 92, -12,
+ -39, -18, 92, -97, -8, -85, -41, 49, -50, 123, -37, -126, -30, 14, 79, -49, -65, 9, -36, -38,
+ -96, 85, -24, -13, 37, -25, -5, -64, -127, 55, -60, -18, -61, -63, 127, 56, 67, 15, 124, 72,
+ 120, 127, 40, -10, 114, 24, -23, 46, 78, -53, 125, 86, 124, 86, 0, 38, 93, 21, 127, 123, 75,
+ -72, 13, 48, 33, 83, -51, 15, -32, -49, -33, 120, 64, 7, 9, 65, 60, 21, -21, -61, -53, -113, 84,
+ -97, 101, 37, -114, -27, 41, 73, 126, -10, 59, 61, -15, 70, -13, 82, -4, 69, 56, 94, -91, -50,
+ 92, -74, -48, 53, -7, -107, 127, 28, 30, -26, -21, -61, 77, 82, 64, -91, -125, 122, -104, 127,
+ 123, 122, 123, 76, -126, 127, -6, -80, 7, 40, -66, -65, 54, -2, 23, 96, -64, 74, 2, -53, -12,
+ -123, 39, 60, -20, 16, -17, -97, 23, -4, -53, -122, 32, -16, -54, -95, 43, 71, -1, -67, -33, 41,
+ 18, 72, 28, -83, 31, -100, -91, -27, 10, -128, -106, 2, 76, -13, 42, 34, 112, -19, 44, 40, -9,
+ -11, 65, 92, -43, -125, 2, 47, -32, 25, 122, -29, 12, 101, -8, -126, -23, 43, 7, 125, -20, -124,
+ 82, -2, 13, -73, -106, 115, 31, 116, -23, -44, -71, 84, 3, 47, 91, 127, 127, -15, 95, 7, 93, 5,
+ 113, -50, 54, 11, 13, -127, 17, 72, 43, -23, 5, -70, 20, 15, -27, 99, 69, -109, -122, -94, 16,
+ 127, 0, 116, 104, 45, 108, -34, 87, 72, -14, 118, 46, 42, 109, -26, 95, 93, 127, 60, 127, -93,
+ -54, -122, 34, -105, 56, 55, 103, 125, -71, -50, 95, -72, 127, 107, 21, 73, 126, 61, 127, 127,
+ 24, -62, 90, 73, 90, -46, -78, -124, 72, 123, -42, 50, -107, 17, -32, -62, -89, 124, 1, 80, -2,
+ 117, 119, -65, -127, -95, -121, -52, 103, 66, 75, -3, -62, -127, 127, -74, 124, 79, 49, 40, 105,
+ -67, -71, -70, 43, 127, 119, -4, 66, 43, 23, 91, -126, 15, 63, -119, 112, 103, 15, -99, 31,
+ -127, 69, 116, -46, -67, 2, -126, -29, 30, 30, -69, -98, -47, -87, -70, -127, 23, -73, 30, -7,
+ 94, -52, -65, 98, -45, 97, 53, 23, -9, -22, -52, -47, 6, -1, -85, -15, -61, -14, 68, 110, -10,
+ -121, -25, -35, -15, -94, -123, 27, 75, 48, -66, -56, -44, 93, 109, 67, -36, 24, 70, -126, 8,
+ -127, 126, 52, 11, -32, 120, -13, -26, -28, -125, 127, 106, -50, 124, 36, -126, -12, 0, -23, 76,
+ -71, -126, -12, -17, -82, 12, 124, 57, 33, 4, 77, -46, 71, -34, 72, 125, -128, 124, -24, -128,
+ 75, -120, 69, -45, 55, 33, 127, -33, 4, -105, -41, -59, -91, 123, 44, -127, 127, -67, 52, 25,
+ -125, -65, 100, -25, 123, 6, 11, -123, -92, -33, 126, -17, -4, 29, 33, 127, 96, 3, 87, -48, -18,
+ -70, 123, 58, -127, -3, -52, -1, -36, -41, 127, 51, -52, -27, 46, -83, 57, 9, 126, 127, 94, 79,
+ -37, -127, -40, 67, 52, 82, -66, 122, -13, -73, 127, -8, -80, 46, -48, 4, -54
+ };
+
+ /** GRU1 bias, [gate][output] (3 x 24). */
+ static final byte[] HIDDEN_GRU_BIAS = {
+ 124, 125, -57, -126, 53, 123, 127, -75, 68, 102, -2, 116, 124, 127, 124, 125, 126, 123, -16, 48,
+ 125, 126, 78, 85, 11, 126, -30, -30, -64, -3, -105, -29, -17, 69, 63, 2, -32, -10, -62, 113,
+ -52, 112, -109, 112, 7, -40, 73, 53, 62, 6, -2, 0, 0, 100, -16, 26, -24, 56, 26, -10, -33, 41,
+ 70, 109, -29, 127, 34, -66, 49, 53, 27, 62
+ };
+
+ /** FC2 weights (24 x 1). */
+ static final byte[] OUTPUT_DENSE_WEIGHTS = {
+ 127, 127, 127, 127, 127, 20, 127, -126, -126, -54, 14, 125, -126, -126, 127, -125, -126, 127,
+ -127, -127, -57, -30, 127, 80
+ };
+
+ /** FC2 bias (1). */
+ static final byte[] OUTPUT_DENSE_BIAS = {
+ -50
+ };
+
+ private RnnVadWeights() {
+ }
+}
diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/vad/SpectralFeatures.java b/ts3-client/core/src/main/java/com/ts3client/audio/vad/SpectralFeatures.java
new file mode 100644
index 0000000..52a5013
--- /dev/null
+++ b/ts3-client/core/src/main/java/com/ts3client/audio/vad/SpectralFeatures.java
@@ -0,0 +1,281 @@
+package com.ts3client.audio.vad;
+
+import static com.ts3client.audio.vad.VadConstants.CEPSTRAL_HISTORY;
+import static com.ts3client.audio.vad.VadConstants.FRAME_20MS_24K;
+import static com.ts3client.audio.vad.VadConstants.NUM_BANDS;
+import static com.ts3client.audio.vad.VadConstants.NUM_LOWER_BANDS;
+import static com.ts3client.audio.vad.VadConstants.OPUS_BANDS_24K;
+
+/**
+ * Band-wise spectral features for {@code rnn_vad}, ported from WebRTC's
+ * {@code rnn_vad/spectral_features.cc} and {@code spectral_features_internal.cc}.
+ *
+ * A 20 ms Vorbis-windowed FFT is folded into 22 Opus-scale bands, log-compressed
+ * and DCT'd into a cepstrum. The network is fed the higher cepstral coefficients, the
+ * running average and first two derivatives of the lower ones, the cepstrum of the
+ * band-wise correlation between the current frame and the frame one pitch period earlier,
+ * and a measure of how much the cepstrum has been varying.
+ */
+final class SpectralFeatures {
+
+ /** Total band energy below which the frame is treated as silence. */
+ private static final float SILENCE_THRESHOLD = 0.04f;
+
+ /** FFT bins per Opus band at 24 kHz for a 20 ms frame. */
+ private static final int[] OPUS_BAND_SIZES =
+ {4, 4, 4, 4, 4, 4, 4, 4, 8, 8, 8, 8, 16, 16, 16, 24, 24, 32, 48};
+
+ /** sqrt(2 / NUM_BANDS), applied after the DCT. */
+ private static final float DCT_SCALING_FACTOR = 0.301511345f;
+
+ private final RealFft fft = new RealFft(FRAME_20MS_24K);
+ private final float[] halfWindow = new float[FRAME_20MS_24K / 2];
+ private final float[] bandWeights = new float[FRAME_20MS_24K / 2];
+ private final float[] dctTable = new float[NUM_BANDS * NUM_BANDS];
+
+ private final float[] windowed = new float[FRAME_20MS_24K];
+ private final float[] referenceFft = new float[FRAME_20MS_24K];
+ private final float[] laggedFft = new float[FRAME_20MS_24K];
+ private final float[] referenceBandsEnergy = new float[OPUS_BANDS_24K];
+ private final float[] laggedBandsEnergy = new float[OPUS_BANDS_24K];
+ private final float[] bandsCrossCorr = new float[OPUS_BANDS_24K];
+ private final float[] logBandsEnergy = new float[NUM_BANDS];
+ private final float[] cepstrum = new float[NUM_BANDS];
+ private final float[] crossCorrCepstrum = new float[NUM_LOWER_BANDS];
+
+ /** Ring buffer of the last {@link VadConstants#CEPSTRAL_HISTORY} cepstra. */
+ private final float[][] cepstralHistory = new float[CEPSTRAL_HISTORY][NUM_BANDS];
+ private int cepstralTail;
+
+ /**
+ * Upper-right triangle of pairwise cepstral distances, stored as a square matrix so a
+ * new frame can be shifted in with a single array copy.
+ */
+ private final float[] cepstralDiffs = new float[(CEPSTRAL_HISTORY - 1) * (CEPSTRAL_HISTORY - 1)];
+ private final float[] newDistances = new float[CEPSTRAL_HISTORY - 1];
+
+ SpectralFeatures() {
+ int halfSize = FRAME_20MS_24K / 2;
+ float scaling = 1.0f / FRAME_20MS_24K;
+ for (int i = 0; i < halfSize; i++) {
+ double inner = Math.sin(0.5 * Math.PI * (i + 0.5) / halfSize);
+ halfWindow[i] = (float) (scaling * Math.sin(0.5 * Math.PI * inner * inner));
+ }
+ // Each band overlaps its neighbour with a triangular response; the weight is the
+ // fractional position of the bin within its band.
+ int bin = 0;
+ for (int bandSize : OPUS_BAND_SIZES) {
+ for (int j = 0; j < bandSize; j++) {
+ bandWeights[bin++] = (float) j / bandSize;
+ }
+ }
+ double k = Math.sqrt(0.5);
+ for (int i = 0; i < NUM_BANDS; i++) {
+ for (int j = 0; j < NUM_BANDS; j++) {
+ dctTable[i * NUM_BANDS + j] = (float) Math.cos((i + 0.5) * j * Math.PI / NUM_BANDS);
+ }
+ dctTable[i * NUM_BANDS] *= (float) k;
+ }
+ }
+
+ void reset() {
+ for (float[] row : cepstralHistory) {
+ java.util.Arrays.fill(row, 0.0f);
+ }
+ cepstralTail = 0;
+ java.util.Arrays.fill(cepstralDiffs, 0.0f);
+ }
+
+ /**
+ * Writes the spectral part of the feature vector and returns {@code true} if the
+ * reference frame is silence, in which case the vector is only partially written.
+ *
+ * @param reference buffer holding the most recent 20 ms at 24 kHz
+ * @param referenceOffset offset of that frame within {@code reference}
+ * @param lagged buffer holding the 20 ms one pitch period earlier
+ * @param laggedOffset offset of that frame within {@code lagged}
+ */
+ boolean checkSilenceComputeFeatures(float[] reference, int referenceOffset,
+ float[] lagged, int laggedOffset,
+ float[] featureVector) {
+ forwardFft(reference, referenceOffset, referenceFft);
+ crossCorrelation(referenceFft, referenceFft, referenceBandsEnergy);
+
+ float totalEnergy = 0;
+ for (float e : referenceBandsEnergy) {
+ totalEnergy += e;
+ }
+ if (totalEnergy < SILENCE_THRESHOLD) {
+ return true;
+ }
+
+ forwardFft(lagged, laggedOffset, laggedFft);
+ crossCorrelation(laggedFft, laggedFft, laggedBandsEnergy);
+
+ smoothedLogMagnitudeSpectrum(referenceBandsEnergy, logBandsEnergy);
+ dct(logBandsEnergy, NUM_BANDS, cepstrum, NUM_BANDS);
+ // Ad-hoc correction terms for the first two cepstral coefficients.
+ cepstrum[0] -= 12.0f;
+ cepstrum[1] -= 4.0f;
+
+ pushCepstrum(cepstrum);
+
+ // Higher bands go in as-is; the lower ones are summarised over time instead.
+ System.arraycopy(cepstrum, NUM_LOWER_BANDS, featureVector, NUM_LOWER_BANDS,
+ NUM_BANDS - NUM_LOWER_BANDS);
+ computeAverageAndDerivatives(featureVector);
+ computeNormalizedCepstralCorrelation(featureVector);
+ featureVector[VadConstants.FEATURE_VECTOR_SIZE - 1] = computeVariability();
+ return false;
+ }
+
+ /** Windows a 20 ms frame and transforms it, zeroing the unused Nyquist coefficient. */
+ private void forwardFft(float[] frame, int offset, float[] out) {
+ int halfSize = FRAME_20MS_24K / 2;
+ for (int i = 0, j = FRAME_20MS_24K - 1; i < halfSize; i++, j--) {
+ windowed[i] = frame[offset + i] * halfWindow[i];
+ windowed[j] = frame[offset + j] * halfWindow[i];
+ }
+ fft.forward(windowed, out);
+ out[1] = 0.0f;
+ }
+
+ /**
+ * Folds the interleaved spectra into Opus bands. Each bin contributes to the band it
+ * sits in and to the next one, split by the triangular weight.
+ */
+ private void crossCorrelation(float[] x, float[] y, float[] out) {
+ int k = 0;
+ out[0] = 0.0f;
+ for (int i = 0; i < OPUS_BANDS_24K - 1; i++) {
+ out[i + 1] = 0.0f;
+ for (int j = 0; j < OPUS_BAND_SIZES[i]; j++) {
+ float v = x[2 * k] * y[2 * k] + x[2 * k + 1] * y[2 * k + 1];
+ float tmp = bandWeights[k] * v;
+ out[i] += v - tmp;
+ out[i + 1] += tmp;
+ k++;
+ }
+ }
+ out[0] *= 2.0f; // The first band only receives half a triangle.
+ }
+
+ /** Log magnitude with smoothing over frequency and a decaying floor. */
+ private static void smoothedLogMagnitudeSpectrum(float[] bandsEnergy, float[] out) {
+ final float logOneByHundred = -2.0f;
+ float logMax = logOneByHundred;
+ float follow = logOneByHundred;
+ for (int i = 0; i < out.length; i++) {
+ float x = (i < bandsEnergy.length)
+ ? (float) Math.log10(0.01f + bandsEnergy[i])
+ : logOneByHundred;
+ x = Math.max(logMax - 7.0f, Math.max(follow - 1.5f, x));
+ logMax = Math.max(logMax, x);
+ follow = Math.max(follow - 1.5f, x);
+ out[i] = x;
+ }
+ }
+
+ /** DCT-II of the first {@code inSize} values of {@code in}, truncated to {@code outSize}. */
+ private void dct(float[] in, int inSize, float[] out, int outSize) {
+ for (int i = 0; i < outSize; i++) {
+ float sum = 0;
+ for (int j = 0; j < inSize; j++) {
+ sum += in[j] * dctTable[j * NUM_BANDS + i];
+ }
+ out[i] = sum * DCT_SCALING_FACTOR;
+ }
+ }
+
+ private void pushCepstrum(float[] newCepstrum) {
+ // Distance from the new cepstrum to each older one, nearest first.
+ for (int i = 0; i < CEPSTRAL_HISTORY - 1; i++) {
+ float[] old = cepstralHistoryAt(i);
+ float sum = 0;
+ for (int k = 0; k < NUM_BANDS; k++) {
+ float c = newCepstrum[k] - old[k];
+ sum += c * c;
+ }
+ newDistances[i] = sum;
+ }
+ System.arraycopy(newCepstrum, 0, cepstralHistory[cepstralTail], 0, NUM_BANDS);
+ cepstralTail = (cepstralTail + 1) % CEPSTRAL_HISTORY;
+ pushCepstralDistances();
+ }
+
+ /** Returns the cepstrum pushed {@code delay} frames ago (0 = most recent). */
+ private float[] cepstralHistoryAt(int delay) {
+ int offset = cepstralTail - 1 - delay;
+ if (offset < 0) {
+ offset += CEPSTRAL_HISTORY;
+ }
+ return cepstralHistory[offset];
+ }
+
+ /** Shifts the distance matrix up-left by one and writes the new column. */
+ private void pushCepstralDistances() {
+ final int s = CEPSTRAL_HISTORY;
+ System.arraycopy(cepstralDiffs, s, cepstralDiffs, 0, cepstralDiffs.length - s);
+ for (int i = 0; i < newDistances.length; i++) {
+ cepstralDiffs[(s - 1 - i) * (s - 1) - 1] = newDistances[i];
+ }
+ }
+
+ private float cepstralDistance(int delay1, int delay2) {
+ int row = CEPSTRAL_HISTORY - 1 - delay1;
+ int col = CEPSTRAL_HISTORY - 1 - delay2;
+ if (row > col) { // Only the upper-right triangle is stored.
+ int tmp = row;
+ row = col;
+ col = tmp;
+ }
+ return cepstralDiffs[row * (CEPSTRAL_HISTORY - 1) + (col - 1)];
+ }
+
+ private void computeAverageAndDerivatives(float[] featureVector) {
+ float[] curr = cepstralHistoryAt(0);
+ float[] prev1 = cepstralHistoryAt(1);
+ float[] prev2 = cepstralHistoryAt(2);
+ for (int i = 0; i < NUM_LOWER_BANDS; i++) {
+ featureVector[i] = curr[i] + prev1[i] + prev2[i]; // kernel [1, 1, 1]
+ featureVector[NUM_BANDS + i] = curr[i] - prev2[i]; // kernel [1, 0, -1]
+ featureVector[NUM_BANDS + NUM_LOWER_BANDS + i] =
+ curr[i] - 2 * prev1[i] + prev2[i]; // kernel [1, -2, 1]
+ }
+ }
+
+ /** Band correlation between the current frame and the one a pitch period earlier. */
+ private void computeNormalizedCepstralCorrelation(float[] featureVector) {
+ crossCorrelation(referenceFft, laggedFft, bandsCrossCorr);
+ for (int i = 0; i < bandsCrossCorr.length; i++) {
+ bandsCrossCorr[i] /= (float) Math.sqrt(
+ 0.001f + referenceBandsEnergy[i] * laggedBandsEnergy[i]);
+ }
+ dct(bandsCrossCorr, OPUS_BANDS_24K, crossCorrCepstrum, NUM_LOWER_BANDS);
+ // Ad-hoc correction terms, as for the frame cepstrum.
+ crossCorrCepstrum[0] -= 1.3f;
+ crossCorrCepstrum[1] -= 0.9f;
+ System.arraycopy(crossCorrCepstrum, 0, featureVector,
+ NUM_BANDS + 2 * NUM_LOWER_BANDS, NUM_LOWER_BANDS);
+ }
+
+ /**
+ * How far the current cepstrum sits from its nearest neighbour in the recent history,
+ * averaged over the history. Steady tones score low, speech scores high.
+ */
+ private float computeVariability() {
+ float variability = 0;
+ for (int delay1 = 0; delay1 < CEPSTRAL_HISTORY; delay1++) {
+ float minDist = Float.MAX_VALUE;
+ for (int delay2 = 0; delay2 < CEPSTRAL_HISTORY; delay2++) {
+ if (delay1 == delay2) {
+ continue; // The distance would be 0.
+ }
+ minDist = Math.min(minDist, cepstralDistance(delay1, delay2));
+ }
+ variability += minDist;
+ }
+ // Normalised against training-set statistics.
+ return variability / CEPSTRAL_HISTORY - 2.1f;
+ }
+}
diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/vad/VadConstants.java b/ts3-client/core/src/main/java/com/ts3client/audio/vad/VadConstants.java
new file mode 100644
index 0000000..a59ac37
--- /dev/null
+++ b/ts3-client/core/src/main/java/com/ts3client/audio/vad/VadConstants.java
@@ -0,0 +1,54 @@
+package com.ts3client.audio.vad;
+
+/**
+ * Sizes and pitch-search bounds of WebRTC's {@code rnn_vad}, mirroring
+ * {@code modules/audio_processing/agc2/rnn_vad/common.h}.
+ *
+ * The detector analyses 10 ms frames at 24 kHz. Spectral analysis uses the
+ * most recent 20 ms; pitch search additionally needs one maximum pitch period of
+ * history, hence the 864-sample buffer.
+ */
+final class VadConstants {
+
+ static final int SAMPLE_RATE_24K = 24_000;
+ static final int FRAME_10MS_24K = SAMPLE_RATE_24K / 100; // 240
+ static final int FRAME_20MS_24K = FRAME_10MS_24K * 2; // 480
+
+ /** Pitch range at 24 kHz: 0.00125 s to 0.016 s. */
+ static final int MIN_PITCH_24K = SAMPLE_RATE_24K / 800; // 30
+ static final int MAX_PITCH_24K = 384; // 24000 / 62.5
+ static final int BUF_SIZE_24K = MAX_PITCH_24K + FRAME_20MS_24K; // 864
+
+ /**
+ * The coarse search skips very short periods; a refinement step recovers them, so the
+ * initial pass starts three minimum periods in.
+ */
+ static final int INITIAL_MIN_PITCH_24K = 3 * MIN_PITCH_24K; // 90
+ static final int INITIAL_NUM_LAGS_24K = MAX_PITCH_24K - INITIAL_MIN_PITCH_24K; // 294
+ static final int REFINE_NUM_LAGS_24K = MAX_PITCH_24K + 1; // 385
+
+ // 12 kHz analysis, used for the coarse pitch search.
+ static final int FRAME_20MS_12K = 240;
+ static final int BUF_SIZE_12K = BUF_SIZE_24K / 2; // 432
+ static final int INITIAL_MIN_PITCH_12K = INITIAL_MIN_PITCH_24K / 2; // 45
+ static final int MAX_PITCH_12K = MAX_PITCH_24K / 2; // 192
+ static final int NUM_LAGS_12K = MAX_PITCH_12K - INITIAL_MIN_PITCH_12K; // 147
+
+ // The pitch period fed to the network is expressed at 48 kHz.
+ static final int MIN_PITCH_48K = MIN_PITCH_24K * 2; // 60
+ static final int MAX_PITCH_48K = MAX_PITCH_24K * 2; // 768
+
+ // Spectral features.
+ static final int NUM_BANDS = 22;
+ static final int NUM_LOWER_BANDS = 6;
+ static final int CEPSTRAL_HISTORY = 8;
+ /** Opus bands that fall below the 24 kHz Nyquist frequency. */
+ static final int OPUS_BANDS_24K = 20;
+
+ static final int NUM_LPC_COEFFICIENTS = 5;
+
+ static final int FEATURE_VECTOR_SIZE = 42;
+
+ private VadConstants() {
+ }
+}
diff --git a/ts3-client/core/src/main/java/com/ts3client/config/Settings.java b/ts3-client/core/src/main/java/com/ts3client/config/Settings.java
index fcc6fec..1e0d0c6 100644
--- a/ts3-client/core/src/main/java/com/ts3client/config/Settings.java
+++ b/ts3-client/core/src/main/java/com/ts3client/config/Settings.java
@@ -69,9 +69,15 @@ public final class Settings {
public InputMode inputMode = InputMode.VOICE_ACTIVATION;
/** Voice-activation strategy used when {@link #inputMode} is VOICE_ACTIVATION. */
public VadMode vadMode = VadMode.HYBRID;
- /** VAD threshold in dBFS (RMS). Typical range -60 (sensitive) .. -10 (loud). */
- public double vadThresholdDb = -45.0;
- /** Speech-probability threshold (0..1) for Automatic/Hybrid modes. */
+ /**
+ * Volume-gate threshold on {@link com.ts3client.audio.InputLevel}'s scale, the same one
+ * the TS3 client's slider uses: -50 (most sensitive) .. +50, defaulting to -40.
+ */
+ public double vadThresholdDb = -40.0;
+ /**
+ * Speech-probability threshold (0..1) for Automatic/Hybrid modes. TS3 calls this
+ * {@code vad_likelihood} and also defaults it to 0.5.
+ */
public double speechThreshold = 0.5;
/** Keep voice activation running while in push-to-talk mode. */
public boolean vadOverPtt = false;
@@ -168,7 +174,8 @@ public final class Settings {
outputDevice = props.getProperty("outputDevice", outputDevice);
inputMode = parseMode(props.getProperty("inputMode"), inputMode);
vadMode = parseVadMode(props.getProperty("vadMode"), vadMode);
- vadThresholdDb = parseD(props.getProperty("vadThresholdDb"), vadThresholdDb);
+ vadThresholdDb = parseD(props.getProperty("vadLevelDb"),
+ migrateThreshold(props.getProperty("vadThresholdDb"), vadThresholdDb));
speechThreshold = parseD(props.getProperty("speechThreshold"), speechThreshold);
vadOverPtt = parseB(props.getProperty("vadOverPtt"), vadOverPtt);
bitrate = parseI(props.getProperty("bitrate"), bitrate);
@@ -205,7 +212,8 @@ public final class Settings {
props.setProperty("outputDevice", outputDevice);
props.setProperty("inputMode", inputMode.name());
props.setProperty("vadMode", vadMode.name());
- props.setProperty("vadThresholdDb", Double.toString(vadThresholdDb));
+ props.setProperty("vadLevelDb", Double.toString(vadThresholdDb));
+ props.remove("vadThresholdDb");
props.setProperty("speechThreshold", Double.toString(speechThreshold));
props.setProperty("vadOverPtt", Boolean.toString(vadOverPtt));
props.setProperty("bitrate", Integer.toString(bitrate));
@@ -246,6 +254,20 @@ public final class Settings {
}
}
+ /**
+ * Converts a threshold saved under the old {@code vadThresholdDb} key, which was in
+ * dBFS ({@code 20*log10(rms)}), to {@link com.ts3client.audio.InputLevel}'s scale, so
+ * upgrading doesn't silently change how sensitive the gate is.
+ */
+ private static double migrateThreshold(String legacy, double def) {
+ double dbfs = parseD(legacy, Double.NaN);
+ if (Double.isNaN(dbfs)) {
+ return def;
+ }
+ double converted = com.ts3client.audio.InputLevel.powerToDb(Math.pow(10.0, dbfs / 10.0));
+ return Math.min(com.ts3client.audio.InputLevel.MAX_DB, converted);
+ }
+
private static double parseD(String v, double def) {
if (v == null) return def;
try {
diff --git a/ts3-client/core/src/test/java/com/ts3client/audio/vad/RealFftTest.java b/ts3-client/core/src/test/java/com/ts3client/audio/vad/RealFftTest.java
new file mode 100644
index 0000000..19c2722
--- /dev/null
+++ b/ts3-client/core/src/test/java/com/ts3client/audio/vad/RealFftTest.java
@@ -0,0 +1,58 @@
+package com.ts3client.audio.vad;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Random;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class RealFftTest {
+
+ /**
+ * The rnn_vad transform length is 480 = 2^5 * 3 * 5, so the mixed-radix path has to be
+ * right for factors other than powers of two.
+ */
+ @Test
+ void matchesDirectDftForMixedRadixLengths() {
+ for (int n : new int[]{480, 240, 512, 64, 60, 210, 100}) {
+ RealFft fft = new RealFft(n);
+ Random random = new Random(n);
+ float[] in = new float[n];
+ float[] out = new float[n];
+ for (int i = 0; i < n; i++) {
+ in[i] = (float) random.nextGaussian();
+ }
+ fft.forward(in, out);
+
+ for (int k = 0; k <= n / 2; k++) {
+ double re = 0;
+ double im = 0;
+ for (int t = 0; t < n; t++) {
+ double angle = -2 * Math.PI * k * t / n;
+ re += in[t] * Math.cos(angle);
+ im += in[t] * Math.sin(angle);
+ }
+ double gotRe;
+ double gotIm;
+ if (k == 0) {
+ gotRe = out[0];
+ gotIm = 0;
+ } else if (k == n / 2) {
+ gotRe = out[1]; // Packed layout keeps Nyquist in slot 1.
+ gotIm = 0;
+ } else {
+ gotRe = out[2 * k];
+ gotIm = out[2 * k + 1];
+ }
+ assertEquals(re, gotRe, 1e-2, "n=" + n + " bin=" + k + " real");
+ assertEquals(im, gotIm, 1e-2, "n=" + n + " bin=" + k + " imag");
+ }
+ }
+ }
+
+ @Test
+ void rejectsOddLengths() {
+ org.junit.jupiter.api.Assertions.assertThrows(
+ IllegalArgumentException.class, () -> new RealFft(481));
+ }
+}
diff --git a/ts3-client/core/src/test/java/com/ts3client/audio/vad/RnnVadTest.java b/ts3-client/core/src/test/java/com/ts3client/audio/vad/RnnVadTest.java
new file mode 100644
index 0000000..c865a97
--- /dev/null
+++ b/ts3-client/core/src/test/java/com/ts3client/audio/vad/RnnVadTest.java
@@ -0,0 +1,129 @@
+package com.ts3client.audio.vad;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Random;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Behavioural checks on the {@code rnn_vad} port. The network weights are the ones the TS3
+ * client links, so these also pin the detector's agreement with it.
+ */
+class RnnVadTest {
+
+ private static final int FRAMES = 60;
+
+ @Test
+ void silenceScoresZero() {
+ float[] x = new float[RnnVad.FRAME_SIZE * FRAMES];
+ double[] probabilities = run(x);
+ for (double p : probabilities) {
+ assertEquals(0.0, p, 0.0, "silence must not score above zero");
+ }
+ }
+
+ @Test
+ void voicedSpeechLikeSignalScoresHigh() {
+ float[] x = synth(FRAMES, t -> {
+ double sum = 0;
+ for (int h = 1; h <= 30 && 120 * h < 8000; h++) {
+ double f = 120 * h;
+ double gain = formant(f, 700, 90) + formant(f, 1220, 110) + formant(f, 2600, 160);
+ sum += gain * Math.sin(2 * Math.PI * f * t + h);
+ }
+ // A slow tremolo keeps it from looking like a stationary tone.
+ return 0.25 * (0.6 + 0.4 * Math.sin(2 * Math.PI * 5 * t)) * sum / 6.0;
+ });
+ // Skip the first frames: the pitch buffer and GRU need history.
+ assertTrue(meanFrom(run(x), 20) > 0.8, "a voiced signal should score high");
+ }
+
+ @Test
+ void quietPinkNoiseScoresLow() {
+ Random random = new Random(3);
+ float[] x = new float[RnnVad.FRAME_SIZE * FRAMES];
+ double b0 = 0;
+ double b1 = 0;
+ double b2 = 0;
+ for (int i = 0; i < x.length; i++) {
+ double w = random.nextGaussian();
+ b0 = 0.99765 * b0 + w * 0.0990460;
+ b1 = 0.96300 * b1 + w * 0.2965164;
+ b2 = 0.57000 * b2 + w * 1.0526913;
+ // About -45 dBFS: representative of room tone rather than loud hiss.
+ x[i] = (float) ((b0 + b1 + b2 + w * 0.1848) * 0.0011);
+ }
+ assertTrue(meanFrom(run(x), 20) < 0.2, "quiet room tone should not read as speech");
+ }
+
+ @Test
+ void estimatesPitchPeriodExactly() {
+ // The pitch feature is fed to the network as 0.01 * (period48k - 300).
+ for (double f0 : new double[]{80, 100, 120, 150, 200, 250, 300, 400}) {
+ RnnVad vad = new RnnVad();
+ float[] x = synth(FRAMES, t -> {
+ double sum = 0;
+ for (int h = 1; h <= 40 && f0 * h < 11000; h++) {
+ sum += Math.sin(2 * Math.PI * f0 * h * t) / h;
+ }
+ return 0.2 * sum;
+ });
+ for (int f = 0; f < FRAMES; f++) {
+ vad.process(x, f * RnnVad.FRAME_SIZE);
+ }
+ assertEquals(48000 / f0, vad.pitchPeriod48kHz(), 1e-9,
+ "pitch period for f0=" + f0);
+ }
+ }
+
+ @Test
+ void resetClearsState() {
+ float[] x = synth(FRAMES, t -> 0.3 * Math.sin(2 * Math.PI * 200 * t));
+ RnnVad vad = new RnnVad();
+ for (int f = 0; f < FRAMES; f++) {
+ vad.process(x, f * RnnVad.FRAME_SIZE);
+ }
+ assertTrue(vad.probability() > 0.0);
+ vad.reset();
+ assertEquals(0.0, vad.probability(), 0.0);
+ }
+
+ // ---- helpers ----
+
+ private interface Signal {
+ double at(double t);
+ }
+
+ private static double formant(double f, double centre, double bandwidth) {
+ double d = (f - centre) / bandwidth;
+ return 1.0 / (1.0 + d * d);
+ }
+
+ private static float[] synth(int frames, Signal signal) {
+ float[] x = new float[RnnVad.FRAME_SIZE * frames];
+ for (int i = 0; i < x.length; i++) {
+ x[i] = (float) signal.at((double) i / RnnVad.SAMPLE_RATE);
+ }
+ return x;
+ }
+
+ private static double[] run(float[] x) {
+ RnnVad vad = new RnnVad();
+ int frames = x.length / RnnVad.FRAME_SIZE;
+ double[] probabilities = new double[frames];
+ for (int i = 0; i < frames; i++) {
+ probabilities[i] = vad.process(x, i * RnnVad.FRAME_SIZE);
+ }
+ return probabilities;
+ }
+
+ private static double meanFrom(double[] values, int start) {
+ double sum = 0;
+ for (int i = start; i < values.length; i++) {
+ sum += values[i];
+ }
+ return sum / (values.length - start);
+ }
+}
diff --git a/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/DesktopVoiceInput.java b/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/DesktopVoiceInput.java
index 9938522..b1c7766 100644
--- a/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/DesktopVoiceInput.java
+++ b/ts3-client/desktop/src/main/java/com/ts3client/audio/desktop/DesktopVoiceInput.java
@@ -2,9 +2,12 @@ package com.ts3client.audio.desktop;
import com.github.manevolent.ts3j.enums.CodecType;
import com.ts3client.audio.AudioEnhancer;
+import com.ts3client.audio.AudioFrameListener;
+import com.ts3client.audio.InputLevel;
import com.ts3client.audio.OpusParameters;
-import com.ts3client.audio.SpeechDetector;
+import com.ts3client.audio.SpeechProbabilityDetector;
import com.ts3client.audio.VoiceInput;
+import com.ts3client.audio.vad.RnnSpeechDetector;
import com.ts3client.config.Settings;
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -28,7 +31,23 @@ import java.util.function.Consumer;
*/
public final class DesktopVoiceInput implements VoiceInput {
- private static final int HANGOVER_FRAMES = 15; // ~300 ms of tail after level drops
+ /**
+ * How long the gate stays open after the last active frame. TS3 counts 64 of its
+ * 10 ms preprocessor frames before closing, so the tail of a word is never clipped.
+ */
+ private static final int HANGOVER_MS = 640;
+
+ /**
+ * Audio replayed when the gate opens, so a word's onset is not swallowed by the frame
+ * that detected it. TS3 keeps up to three 10 ms buffers ({@code vad_extrabuffersize}
+ * defaults to 2, plus one).
+ */
+ private static final int PREROLL_MS = 30;
+
+ private static final int HANGOVER_FRAMES =
+ Math.max(1, HANGOVER_MS * AudioDevices.SAMPLE_RATE / 1000 / AudioDevices.FRAME_SIZE);
+ private static final int PREROLL_FRAMES =
+ Math.max(1, PREROLL_MS * AudioDevices.SAMPLE_RATE / 1000 / AudioDevices.FRAME_SIZE);
private final ConcurrentLinkedQueue The modes match the TS3 client's: the volume gate alone, the speech detector
+ * alone, or both together. Volume Gate skips the detector entirely, as TS3 does.
*/
private boolean voiceActivated(double db, float[] pcm) {
boolean detected;
@@ -363,11 +436,11 @@ public final class DesktopVoiceInput implements VoiceInput {
detected = db >= thresholdDb;
break;
case AUTOMATIC:
- detected = speechDetector.process(pcm) >= speechThreshold;
+ detected = speechDetector.process(pcm, AudioDevices.FRAME_SIZE) >= speechThreshold;
break;
case HYBRID:
default:
- double probability = speechDetector.process(pcm);
+ double probability = speechDetector.process(pcm, AudioDevices.FRAME_SIZE);
detected = db >= thresholdDb && probability >= speechThreshold;
break;
}
@@ -395,6 +468,12 @@ public final class DesktopVoiceInput implements VoiceInput {
mutedTalking = talking;
}
+ /** Hands a transmitted frame to the monitor, if one is attached. */
+ private void monitor(float[] frame, int channels) {
+ AudioFrameListener l = monitorListener;
+ if (l != null) l.onFrame(frame, channels);
+ }
+
private void setTransmitting(boolean t) {
transmitting.set(t);
if (t != lastTransmitting) {
diff --git a/ts3-client/swing/pom.xml b/ts3-client/swing/pom.xml
index 70a17b0..ad09dbd 100644
--- a/ts3-client/swing/pom.xml
+++ b/ts3-client/swing/pom.xml
@@ -35,6 +35,12 @@
Scaled to {@link com.ts3client.audio.InputLevel}, matching the TS3 client's slider.
*/
public final class LevelMeter extends JComponent {
- private static final double MIN_DB = -70.0;
- private static final double MAX_DB = 0.0;
+ private static final double MIN_DB = com.ts3client.audio.InputLevel.MIN_DB;
+ private static final double MAX_DB = com.ts3client.audio.InputLevel.MAX_DB;
private volatile double levelDb = MIN_DB;
- private volatile double thresholdDb = -45.0;
+ private volatile double thresholdDb = -40.0;
private volatile boolean showThreshold = true;
+ private volatile boolean transmitting;
public LevelMeter() {
setPreferredSize(new Dimension(240, 18));
+ // A bare JComponent reports no minimum of its own, so a layout tight on space
+ // collapses the bar to nothing. Keep the height and let only the width give.
+ setMinimumSize(new Dimension(60, 18));
+ setMaximumSize(new Dimension(Integer.MAX_VALUE, 18));
}
public void setLevel(double db) {
@@ -40,6 +47,16 @@ public final class LevelMeter extends JComponent {
repaint();
}
+ /**
+ * Whether the gate is currently open. This is the real transmit decision, which in the
+ * Automatic and Hybrid modes depends on the speech detector and the hangover as well as
+ * on the level, so the bar cannot infer it from the threshold alone.
+ */
+ public void setTransmitting(boolean transmitting) {
+ this.transmitting = transmitting;
+ repaint();
+ }
+
private int dbToX(double db, int w) {
double clamped = Math.max(MIN_DB, Math.min(MAX_DB, db));
return (int) ((clamped - MIN_DB) / (MAX_DB - MIN_DB) * w);
@@ -56,8 +73,7 @@ public final class LevelMeter extends JComponent {
g.fillRoundRect(0, 0, w - 1, h - 1, 6, 6);
int level = dbToX(levelDb, w - 2);
- boolean over = levelDb >= thresholdDb;
- g.setColor(over ? Theme.TALKING : new Color(0x5A9BD4));
+ g.setColor(transmitting ? Theme.TALKING : new Color(0x5A9BD4));
g.fillRoundRect(1, 1, Math.max(0, level), h - 3, 5, 5);
if (showThreshold) {
diff --git a/ts3-client/swing/src/main/java/com/ts3client/ui/MicrophoneTest.java b/ts3-client/swing/src/main/java/com/ts3client/ui/MicrophoneTest.java
new file mode 100644
index 0000000..930e76b
--- /dev/null
+++ b/ts3-client/swing/src/main/java/com/ts3client/ui/MicrophoneTest.java
@@ -0,0 +1,181 @@
+package com.ts3client.ui;
+
+import com.ts3client.audio.VoiceInput;
+import com.ts3client.audio.desktop.AudioDevices;
+import com.ts3client.audio.desktop.AudioPlayback;
+import com.ts3client.audio.desktop.DesktopVoiceInput;
+import com.ts3client.config.Settings;
+
+import javax.swing.SwingUtilities;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.function.Consumer;
+
+/**
+ * Drives the settings dialog's microphone test from a real capture chain.
+ *
+ * The dialog runs its own {@link DesktopVoiceInput} rather than borrowing the connected
+ * one, whose listeners belong to the connection. Because it is the same class that feeds
+ * the server, the level and the gate shown here are exactly what would be transmitted —
+ * pre-processing, voice detection, hangover and pre-roll included.
+ *
+ * Loopback is optional: when enabled, the frames that would be sent are played back
+ * locally so you can hear precisely what the other side would.
+ */
+final class MicrophoneTest {
+
+ /** Frames buffered for loopback before the oldest is dropped (~100 ms). */
+ private static final int LOOPBACK_QUEUE_FRAMES = 5;
+
+ private final Consumer The test always runs voice activation: it exists to tune the gate, and push-to-talk
+ * would need the global hotkey, which belongs to the connected microphone.
+ */
+ private Settings audioSnapshot() {
+ Settings snapshot = new Settings();
+ writeAudioSettings(snapshot);
+ if (snapshot.inputMode == Settings.InputMode.PUSH_TO_TALK) {
+ snapshot.inputMode = Settings.InputMode.VOICE_ACTIVATION;
+ }
+ return snapshot;
+ }
+
+ private void apply() {
+ writeAudioSettings(settings);
notificationsPanel.apply();
iconPackPanel.apply();
clientVersionPanel.apply();
hotkeysPanel.apply();
settings.save();
- if (liveMic != null) {
- liveMic.setMode(settings.inputMode);
- liveMic.setVadMode(settings.vadMode);
- liveMic.setThresholdDb(settings.vadThresholdDb);
- liveMic.setSpeechThreshold(settings.speechThreshold);
- liveMic.setVadOverPtt(settings.vadOverPtt);
- liveMic.setInputGain(settings.inputVolume);
- liveMic.setNoiseSuppression(settings.denoise);
- liveMic.setDenoiserLevel(settings.denoiserLevel);
- liveMic.setTypingAttenuation(settings.typingAttenuation);
- liveMic.setAgc(settings.agc);
- liveMic.setOpusParameters(OpusParameters.from(settings));
- }
+ applyLive(m -> {
+ m.setMode(settings.inputMode);
+ m.setVadMode(settings.vadMode);
+ m.setThresholdDb(settings.vadThresholdDb);
+ m.setSpeechThreshold(settings.speechThreshold);
+ m.setVadOverPtt(settings.vadOverPtt);
+ m.setInputGain(settings.inputVolume);
+ m.setNoiseSuppression(settings.denoise);
+ m.setDenoiserLevel(settings.denoiserLevel);
+ m.setTypingAttenuation(settings.typingAttenuation);
+ m.setAgc(settings.agc);
+ m.setOpusParameters(OpusParameters.from(settings));
+ });
if (livePlayback != null) {
livePlayback.setMasterVolume(settings.outputVolume);
livePlayback.setOutputDevice(settings.outputDevice);
@@ -597,64 +625,91 @@ public final class SettingsDialog extends JDialog {
}
private void close() {
- stopMeter();
+ micTest.stop();
dispose();
}
- // ---- live meter ----
+ // ---- microphone test ----
- private void startMeter() {
- meterRunning = true;
- meterThread = new Thread(this::meterLoop, "settings-meter");
- meterThread.setDaemon(true);
- meterThread.start();
+ /**
+ * The test row: a toggle that runs the capture chain, an indicator showing whether the
+ * gate is open, and an optional loopback so you can hear what is being sent.
+ */
+ private JPanel buildTestControls() {
+ JPanel row = new JPanel(new java.awt.FlowLayout(java.awt.FlowLayout.LEFT, 8, 0));
+
+ testButton = new JToggleButton("Begin Test");
+ testButton.setToolTipText("Run the capture chain exactly as it runs while connected, "
+ + "so the bar and the indicator show what would actually be transmitted.");
+ testButton.addActionListener(e -> setTesting(testButton.isSelected()));
+
+ loopbackCheck = new JCheckBox("Hear myself");
+ loopbackCheck.setToolTipText("Play the transmitted audio back through the playback "
+ + "device. Use headphones to avoid feedback.");
+ loopbackCheck.setEnabled(false);
+ loopbackCheck.addActionListener(e -> micTest.setLoopback(loopbackCheck.isSelected()));
+
+ talkIndicator = new JLabel("Not transmitting", Icons.clientIdle(), JLabel.LEFT);
+ talkIndicator.setToolTipText("Lights up while your microphone is open, exactly as "
+ + "other users would see you in the channel list.");
+
+ row.add(testButton);
+ row.add(loopbackCheck);
+ row.add(talkIndicator);
+ return row;
}
- private void restartMeter() {
- stopMeter();
- startMeter();
+ private void setTesting(boolean on) {
+ if (on && !micTest.start(audioSnapshot())) {
+ testButton.setSelected(false);
+ testButton.setText("Begin Test");
+ loopbackCheck.setEnabled(false);
+ resetTestIndicators();
+ talkIndicator.setText("Capture device unavailable");
+ return;
+ }
+ if (!on) {
+ micTest.stop();
+ loopbackCheck.setSelected(false);
+ resetTestIndicators();
+ }
+ testButton.setText(on ? "Stop Test" : "Begin Test");
+ loopbackCheck.setEnabled(on);
}
- private void stopMeter() {
- meterRunning = false;
- if (meterThread != null) {
- meterThread.interrupt();
- meterThread = null;
+ private void resetTestIndicators() {
+ onTestTalking(false);
+ onTestLevel(InputLevel.SILENCE_DB);
+ }
+
+ /** Restarts the test chain, if running, so a device change takes effect. */
+ private void restartTest() {
+ if (micTest.isRunning()) {
+ boolean loopback = loopbackCheck.isSelected();
+ if (micTest.start(audioSnapshot())) {
+ micTest.setLoopback(loopback);
+ } else {
+ setTesting(false);
+ testButton.setSelected(false);
+ }
}
}
- private void meterLoop() {
- String device = comboValue(inputCombo);
- AudioCapture line = null;
- try {
- line = AudioDevices.openCapture(device, AudioDevices.MAX_CHANNELS);
- line.start();
- int frame = AudioDevices.FRAME_SIZE;
- int channels = line.channels();
- byte[] buf = new byte[frame * 2 * channels];
- double gain = (inputGain != null ? inputGain.getValue() / 100.0 : 1.0);
- while (meterRunning) {
- if (line.read(buf, 0, buf.length) < buf.length) break;
- // Metering follows the capture chain: level is taken off the downmix.
- double sumSq = 0;
- for (int i = 0; i < frame; i++) {
- double mono = 0;
- for (int c = 0; c < channels; c++) {
- int k = i * channels + c;
- short s = (short) ((buf[2 * k + 1] << 8) | (buf[2 * k] & 0xFF));
- mono += s / 32768.0;
- }
- double f = mono / channels * gain;
- sumSq += f * f;
- }
- double rms = Math.sqrt(sumSq / frame);
- double db = rms <= 1e-9 ? -100 : 20 * Math.log10(rms);
- final double fdb = db;
- if (meter != null) SwingUtilities.invokeLater(() -> meter.setLevel(fdb));
- }
- } catch (Exception ignored) {
- } finally {
- if (line != null) line.close();
+ /** Applies a live change to the connected microphone and to the test one alike. */
+ private void applyLive(java.util.function.Consumer
+ * out[0] = Re(X[0]) (DC)
+ * out[1] = Re(X[n/2]) (Nyquist)
+ * out[2k] = Re(X[k])
+ * out[2k+1] = Im(X[k]) for k in [1, n/2)
+ *
+ */
+final class RealFft {
+
+ private final int size;
+ private final int half;
+
+ // Twiddles for the complex sub-transform and for the real split step.
+ private final float[] halfRe;
+ private final float[] halfIm;
+ private final float[] splitCos;
+ private final float[] splitSin;
+
+ private final ComplexFft complexFft;
+
+ RealFft(int size) {
+ if (size < 2 || (size & 1) != 0) {
+ throw new IllegalArgumentException("Real FFT length must be even: " + size);
+ }
+ this.size = size;
+ this.half = size / 2;
+ this.halfRe = new float[half];
+ this.halfIm = new float[half];
+ this.complexFft = new ComplexFft(half);
+
+ this.splitCos = new float[half / 2 + 1];
+ this.splitSin = new float[half / 2 + 1];
+ for (int k = 0; k <= half / 2; k++) {
+ double angle = -2.0 * Math.PI * k / size;
+ splitCos[k] = (float) Math.cos(angle);
+ splitSin[k] = (float) Math.sin(angle);
+ }
+ }
+
+ int size() {
+ return size;
+ }
+
+ /**
+ * Transforms {@code in} ({@link #size} real samples) into {@code out}
+ * ({@link #size} floats in the packed layout described above).
+ */
+ void forward(float[] in, float[] out) {
+ // Pack the real signal as half/2 complex samples: z[i] = x[2i] + j*x[2i+1].
+ for (int i = 0; i < half; i++) {
+ halfRe[i] = in[2 * i];
+ halfIm[i] = in[2 * i + 1];
+ }
+ complexFft.forward(halfRe, halfIm);
+
+ // Split Z into the transforms of the even and odd sample sequences, then
+ // recombine them into the spectrum of the original real signal.
+ out[0] = halfRe[0] + halfIm[0];
+ out[1] = halfRe[0] - halfIm[0];
+ for (int k = 1; k <= half / 2; k++) {
+ int mirror = half - k;
+
+ // Even part: hermitian-symmetric average; odd part: the difference.
+ float evenRe = 0.5f * (halfRe[k] + halfRe[mirror]);
+ float evenIm = 0.5f * (halfIm[k] - halfIm[mirror]);
+ float oddRe = 0.5f * (halfIm[k] + halfIm[mirror]);
+ float oddIm = -0.5f * (halfRe[k] - halfRe[mirror]);
+
+ float c = splitCos[k];
+ float s = splitSin[k];
+ float twRe = oddRe * c - oddIm * s;
+ float twIm = oddRe * s + oddIm * c;
+
+ float re = evenRe + twRe;
+ float im = evenIm + twIm;
+ out[2 * k] = re;
+ out[2 * k + 1] = im;
+
+ if (mirror != k) {
+ // X[n/2 - k] is the conjugate-mirrored companion of X[k].
+ out[2 * mirror] = evenRe - twRe;
+ out[2 * mirror + 1] = -(evenIm - twIm);
+ }
+ }
+ }
+
+ /**
+ * Mixed-radix complex FFT, decimation in time.
+ *
+ *
"
+ "Volume Gate: transmit when loud enough.
"
+ "Hybrid: loud enough and detected as speech.");
addRow(p, c, row++, new JLabel("Detection:"), vadModeCombo);
- thresholdSlider = new JSlider(-70, 0, (int) Math.round(settings.vadThresholdDb));
+ thresholdSlider = new JSlider((int) InputLevel.MIN_DB, (int) InputLevel.MAX_DB,
+ (int) Math.round(settings.vadThresholdDb));
thresholdLabel = new JLabel(Math.round(settings.vadThresholdDb) + " dB");
thresholdSlider.addChangeListener(e -> {
meter.setThreshold(thresholdSlider.getValue());
thresholdLabel.setText(thresholdSlider.getValue() + " dB");
- if (liveMic != null) liveMic.setThresholdDb(thresholdSlider.getValue());
+ applyLive(m -> m.setThresholdDb(thresholdSlider.getValue()));
});
addRow(p, c, row++, new JLabel("Volume gate:"), sliderWithLabel(thresholdSlider, thresholdLabel));
@@ -312,7 +317,7 @@ public final class SettingsDialog extends JDialog {
speechLabel = new JLabel(Math.round(settings.speechThreshold * 100) + "%");
speechSlider.addChangeListener(e -> {
speechLabel.setText(speechSlider.getValue() + "%");
- if (liveMic != null) liveMic.setSpeechThreshold(speechSlider.getValue() / 100.0);
+ applyLive(m -> m.setSpeechThreshold(speechSlider.getValue() / 100.0));
});
addRow(p, c, row++, new JLabel("Speech threshold:"), sliderWithLabel(speechSlider, speechLabel));
@@ -322,6 +327,8 @@ public final class SettingsDialog extends JDialog {
addRow(p, c, row++, new JLabel("Push-to-talk hotkey:"), pttKeyButton);
vadOverPttCheck = new JCheckBox("Also detect voice while push-to-talk", settings.vadOverPtt);
+ vadOverPttCheck.addActionListener(e ->
+ micTest.configure(m -> m.setVadOverPtt(vadOverPttCheck.isSelected())));
c.gridx = 0;
c.gridy = row++;
c.gridwidth = 2;
@@ -542,46 +549,67 @@ public final class SettingsDialog extends JDialog {
hotkeysPanel.reload();
}
- private void apply() {
- settings.inputDevice = comboValue(inputCombo);
- settings.outputDevice = comboValue(outputCombo);
- settings.inputVolume = inputGain.getValue() / 100.0;
- settings.outputVolume = outputVol.getValue() / 100.0;
- settings.denoise = denoiseCheck.isSelected();
- settings.denoiserLevel = denoiseLevel.getValue() / 100.0;
- settings.typingAttenuation = typingCheck.isSelected();
- settings.agc = agcCheck.isSelected();
- settings.inputMode = pttRadio.isSelected() ? Settings.InputMode.PUSH_TO_TALK
+ /** Copies the audio form into {@code target}, without touching anything else. */
+ private void writeAudioSettings(Settings target) {
+ target.inputDevice = comboValue(inputCombo);
+ target.outputDevice = comboValue(outputCombo);
+ target.inputVolume = inputGain.getValue() / 100.0;
+ target.outputVolume = outputVol.getValue() / 100.0;
+ target.denoise = denoiseCheck.isSelected();
+ target.denoiserLevel = denoiseLevel.getValue() / 100.0;
+ target.typingAttenuation = typingCheck.isSelected();
+ target.agc = agcCheck.isSelected();
+ target.inputMode = pttRadio.isSelected() ? Settings.InputMode.PUSH_TO_TALK
: contRadio.isSelected() ? Settings.InputMode.CONTINUOUS
: Settings.InputMode.VOICE_ACTIVATION;
- settings.vadMode = currentVadMode();
- settings.vadThresholdDb = thresholdSlider.getValue();
- settings.speechThreshold = speechSlider.getValue() / 100.0;
- settings.vadOverPtt = vadOverPttCheck.isSelected();
- settings.bitrate = bitrateSlider.getValue() * 1000;
- settings.complexity = complexitySlider.getValue();
- settings.vbr = vbrCheck.isSelected();
- settings.fec = fecCheck.isSelected();
- settings.music = musicCheck.isSelected();
+ target.vadMode = currentVadMode();
+ target.vadThresholdDb = thresholdSlider.getValue();
+ target.speechThreshold = speechSlider.getValue() / 100.0;
+ target.vadOverPtt = vadOverPttCheck.isSelected();
+ target.bitrate = bitrateSlider.getValue() * 1000;
+ target.complexity = complexitySlider.getValue();
+ target.vbr = vbrCheck.isSelected();
+ target.fec = fecCheck.isSelected();
+ target.music = musicCheck.isSelected();
+ }
+
+ /**
+ * The audio form as a standalone {@link Settings}, so the test chain runs with the
+ * values currently on screen rather than the ones last saved.
+ *
+ *