Replace RMS voice detection with TeamSpeak's own RNN VAD
Ports WebRTC's rnn_vad (as TS3 embeds it) to Java: LPC, pitch estimation, spectral features and the RNN itself, feeding a speech-probability detector that replaces the old SpeechDetector. Also switches the volume-gate threshold from raw dBFS to InputLevel's scale, matching TS3's own slider and range, with a migration for settings saved under the old key.
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
package com.ts3client.audio;
|
||||
|
||||
/** Receives frames of PCM audio as they pass through the capture or playback chain. */
|
||||
@FunctionalInterface
|
||||
public interface AudioFrameListener {
|
||||
|
||||
/**
|
||||
* Called with one frame of interleaved samples in [-1, 1].
|
||||
*
|
||||
* <p>The array is reused between calls, so copy anything you need to keep. Called from
|
||||
* the audio thread: do not block.
|
||||
*
|
||||
* @param interleaved samples, {@code channels} per sample position
|
||||
* @param channels number of interleaved channels
|
||||
*/
|
||||
void onFrame(float[] interleaved, int channels);
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package com.ts3client.audio;
|
||||
|
||||
/**
|
||||
* Capture level on the scale the TeamSpeak 3 client puts on its voice-activation slider.
|
||||
*
|
||||
* <p>TS3 stores {@code voiceactivation_level} in decibels but compares powers: its setter
|
||||
* converts the slider value with {@code 10^(dB/10) * 0.0095} and its getter reports a
|
||||
* measured power back with {@code 10*log10(power / 0.0095)}. The same reference constant is
|
||||
* used here so a threshold of −40 means the same thing it does in TeamSpeak.
|
||||
*
|
||||
* <p>The resulting scale runs from {@link #MIN_DB} to {@link #MAX_DB}, with 0 dB
|
||||
* corresponding to roughly −20 dBFS and the default −40 to about
|
||||
* −60 dBFS.
|
||||
*/
|
||||
public final class InputLevel {
|
||||
|
||||
/**
|
||||
* Mean-square level that reads as 0 dB, taken from the client's own dB conversion.
|
||||
*/
|
||||
private static final double REFERENCE_POWER = 0.0095;
|
||||
|
||||
/** Lower end of the slider range. */
|
||||
public static final double MIN_DB = -50.0;
|
||||
|
||||
/** Upper end of the slider range. */
|
||||
public static final double MAX_DB = 50.0;
|
||||
|
||||
/** Reported when the frame is digital silence. */
|
||||
public static final double SILENCE_DB = MIN_DB;
|
||||
|
||||
private InputLevel() {
|
||||
}
|
||||
|
||||
/** Level of {@code length} mono samples in [-1, 1], in dB on the scale above. */
|
||||
public static double toDb(float[] frame, int length) {
|
||||
if (length <= 0) {
|
||||
return SILENCE_DB;
|
||||
}
|
||||
double sumSq = 0;
|
||||
for (int i = 0; i < length; i++) {
|
||||
sumSq += (double) frame[i] * frame[i];
|
||||
}
|
||||
return powerToDb(sumSq / length);
|
||||
}
|
||||
|
||||
/** Converts a mean-square power to the slider's dB scale. */
|
||||
public static double powerToDb(double power) {
|
||||
if (power <= 0) {
|
||||
return SILENCE_DB;
|
||||
}
|
||||
return Math.max(SILENCE_DB, 10.0 * Math.log10(power / REFERENCE_POWER));
|
||||
}
|
||||
|
||||
/** Inverse of {@link #powerToDb}, matching the client's threshold conversion. */
|
||||
public static double dbToPower(double db) {
|
||||
return Math.pow(10.0, db / 10.0) * REFERENCE_POWER;
|
||||
}
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package com.ts3client.audio;
|
||||
|
||||
/**
|
||||
* Lightweight speech-presence detector used by the "Automatic" and "Hybrid" voice
|
||||
* activation modes (the TeamSpeak 3 client uses a WebRTC GMM detector for the same
|
||||
* purpose). It classifies each 20 ms frame from three features — short-term
|
||||
* energy, spectral flatness and dominant frequency — against an adaptive noise
|
||||
* floor, following Moattar & Homayounpour's real-time VAD.
|
||||
*
|
||||
* <p>Pure DSP with no platform dependencies, so any frontend/backend can reuse it.
|
||||
* A frame votes for speech when at least two of the three features exceed the
|
||||
* baseline; the smoothed vote fraction is exposed as a [0,1] probability.
|
||||
*/
|
||||
public final class SpeechDetector {
|
||||
|
||||
private static final int FFT_SIZE = 1024;
|
||||
|
||||
// Primary thresholds from the reference algorithm (16-bit sample scale).
|
||||
private static final double ENERGY_PRIM = 40.0;
|
||||
private static final double DOMINANT_FREQ_PRIM = 185.0;
|
||||
private static final double FLATNESS_PRIM = 5.0;
|
||||
|
||||
private final int sampleRate;
|
||||
private final double[] re = new double[FFT_SIZE];
|
||||
private final double[] im = new double[FFT_SIZE];
|
||||
|
||||
private double minEnergy;
|
||||
private double minDominantFreq;
|
||||
private double minFlatness;
|
||||
private long silenceFrames;
|
||||
private long frameCount;
|
||||
private double probability;
|
||||
|
||||
public SpeechDetector(int sampleRate) {
|
||||
this.sampleRate = sampleRate;
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
frameCount = 0;
|
||||
silenceFrames = 0;
|
||||
probability = 0;
|
||||
}
|
||||
|
||||
public double getProbability() {
|
||||
return probability;
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes one frame of mono PCM in [-1,1] and returns the smoothed speech
|
||||
* probability in [0,1].
|
||||
*/
|
||||
public double process(float[] frame) {
|
||||
double energy = 0;
|
||||
int n = Math.min(frame.length, FFT_SIZE);
|
||||
for (int i = 0; i < n; i++) {
|
||||
double s = frame[i] * 32768.0; // emulate 16-bit scale
|
||||
energy += s * s;
|
||||
re[i] = frame[i];
|
||||
im[i] = 0;
|
||||
}
|
||||
for (int i = n; i < FFT_SIZE; i++) {
|
||||
re[i] = 0;
|
||||
im[i] = 0;
|
||||
}
|
||||
|
||||
Fft.forward(re, im);
|
||||
|
||||
int half = FFT_SIZE / 2;
|
||||
double geoLogSum = 0;
|
||||
double arithSum = 0;
|
||||
double maxMag = 0;
|
||||
int maxBin = 0;
|
||||
for (int k = 1; k < half; k++) {
|
||||
double mag = Math.sqrt(re[k] * re[k] + im[k] * im[k]) + 1e-12;
|
||||
geoLogSum += Math.log(mag);
|
||||
arithSum += mag;
|
||||
if (mag > maxMag) {
|
||||
maxMag = mag;
|
||||
maxBin = k;
|
||||
}
|
||||
}
|
||||
int bins = half - 1;
|
||||
double geoMean = Math.exp(geoLogSum / bins);
|
||||
double arithMean = arithSum / bins;
|
||||
double flatness = -10.0 * Math.log10(geoMean / arithMean); // high = tonal (speech)
|
||||
double dominantFreq = (double) maxBin * sampleRate / FFT_SIZE;
|
||||
|
||||
if (frameCount == 0) {
|
||||
minEnergy = Math.max(energy, 1.0);
|
||||
minDominantFreq = dominantFreq;
|
||||
minFlatness = flatness;
|
||||
}
|
||||
|
||||
double energyThresh = ENERGY_PRIM * Math.log10(Math.max(minEnergy, 1.0));
|
||||
int votes = 0;
|
||||
if (energy - minEnergy >= energyThresh) votes++;
|
||||
if (dominantFreq - minDominantFreq >= DOMINANT_FREQ_PRIM) votes++;
|
||||
if (flatness - minFlatness >= FLATNESS_PRIM) votes++;
|
||||
|
||||
boolean speech = votes >= 2;
|
||||
if (!speech) {
|
||||
// Adapt the noise floor towards the current (silent) energy.
|
||||
silenceFrames++;
|
||||
minEnergy = ((silenceFrames * minEnergy) + energy) / (silenceFrames + 1);
|
||||
}
|
||||
|
||||
double instant = votes / 3.0;
|
||||
probability = 0.6 * probability + 0.4 * instant;
|
||||
frameCount++;
|
||||
return probability;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package com.ts3client.audio;
|
||||
|
||||
/**
|
||||
* Estimates how likely it is that a frame of captured audio contains speech.
|
||||
*
|
||||
* <p>Used by the "Automatic" and "Hybrid" voice-activation modes. Implementations are
|
||||
* stateful and expect frames in capture order; {@link #reset()} clears that state.
|
||||
*
|
||||
* <p>Pure DSP with no platform dependencies, so any frontend or backend can reuse it.
|
||||
*/
|
||||
public interface SpeechProbabilityDetector {
|
||||
|
||||
/**
|
||||
* Consumes one frame of mono PCM in [-1, 1] and returns the current speech
|
||||
* probability in [0, 1].
|
||||
*
|
||||
* @param frame samples in capture order
|
||||
* @param length number of valid samples in {@code frame}
|
||||
*/
|
||||
double process(float[] frame, int length);
|
||||
|
||||
/** Speech probability from the most recent {@link #process} call. */
|
||||
double probability();
|
||||
|
||||
/** Drops all history. Call on a capture discontinuity or a settings change. */
|
||||
void reset();
|
||||
}
|
||||
@@ -22,7 +22,7 @@ public interface VoiceInput extends Microphone {
|
||||
|
||||
void setVadMode(Settings.VadMode mode);
|
||||
|
||||
/** Voice-activation volume-gate threshold in dBFS. */
|
||||
/** Volume-gate threshold on {@link InputLevel}'s scale. */
|
||||
void setThresholdDb(double db);
|
||||
|
||||
/** Speech-probability threshold (0..1) for Automatic/Hybrid modes. */
|
||||
@@ -50,12 +50,19 @@ public interface VoiceInput extends Microphone {
|
||||
|
||||
void setPushToTalk(boolean pressed);
|
||||
|
||||
/** Receives the live input level in dBFS, once per captured frame. */
|
||||
/** Receives the live input level on {@link InputLevel}'s scale, once per captured frame. */
|
||||
void setLevelListener(Consumer<Double> listener);
|
||||
|
||||
/** Receives local transmit-state transitions (talking / silent). */
|
||||
void setTalkListener(Consumer<Boolean> listener);
|
||||
|
||||
/**
|
||||
* Receives every frame that is actually transmitted, for local monitoring: the audio
|
||||
* after pre-processing and gating, including the pre-roll replayed when the gate opens.
|
||||
* Nothing is delivered while the gate is shut or the microphone is muted.
|
||||
*/
|
||||
void setMonitorListener(AudioFrameListener listener);
|
||||
|
||||
/** Called when speech is detected while the microphone is muted. */
|
||||
void setMutedTalkListener(Runnable listener);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package com.ts3client.audio.vad;
|
||||
|
||||
import static com.ts3client.audio.vad.VadConstants.NUM_LPC_COEFFICIENTS;
|
||||
|
||||
/**
|
||||
* Linear-prediction analysis for the pitch search, ported from WebRTC's
|
||||
* {@code rnn_vad/lp_residual.cc}.
|
||||
*
|
||||
* <p>Whitening the signal before searching for a pitch makes the auto-correlation peak at
|
||||
* the true period rather than at whichever formant happens to dominate.
|
||||
*/
|
||||
final class LinearPrediction {
|
||||
|
||||
private final float[] autoCorr = new float[NUM_LPC_COEFFICIENTS];
|
||||
private final float[] reflection = new float[NUM_LPC_COEFFICIENTS - 1];
|
||||
|
||||
/** Computes the 5 post-processed inverse-filter coefficients for {@code x}. */
|
||||
void computeLpcCoefficients(float[] x, float[] lpcCoeffs) {
|
||||
for (int lag = 0; lag < NUM_LPC_COEFFICIENTS; lag++) {
|
||||
float sum = 0;
|
||||
for (int i = 0; i < x.length - lag; i++) {
|
||||
sum += x[i] * x[i + lag];
|
||||
}
|
||||
autoCorr[lag] = sum;
|
||||
}
|
||||
if (autoCorr[0] == 0.0f) { // Empty frame.
|
||||
java.util.Arrays.fill(lpcCoeffs, 0.0f);
|
||||
return;
|
||||
}
|
||||
|
||||
// Assume a -40 dB white noise floor.
|
||||
autoCorr[0] *= 1.0001f;
|
||||
autoCorr[1] -= autoCorr[1] * 0.000064f;
|
||||
autoCorr[2] -= autoCorr[2] * 0.000256f;
|
||||
autoCorr[3] -= autoCorr[3] * 0.000576f;
|
||||
autoCorr[4] -= autoCorr[4] * 0.001024f;
|
||||
|
||||
java.util.Arrays.fill(reflection, 0.0f);
|
||||
computeInitialInverseFilterCoefficients();
|
||||
|
||||
// Bandwidth expansion, then convolution with (1 + 0.8 z^-1).
|
||||
reflection[0] *= 0.9f;
|
||||
reflection[1] *= 0.9f * 0.9f;
|
||||
reflection[2] *= 0.9f * 0.9f * 0.9f;
|
||||
reflection[3] *= 0.9f * 0.9f * 0.9f * 0.9f;
|
||||
final float c = 0.8f;
|
||||
lpcCoeffs[0] = reflection[0] + c;
|
||||
lpcCoeffs[1] = reflection[1] + c * reflection[0];
|
||||
lpcCoeffs[2] = reflection[2] + c * reflection[1];
|
||||
lpcCoeffs[3] = reflection[3] + c * reflection[2];
|
||||
lpcCoeffs[4] = c * reflection[3];
|
||||
}
|
||||
|
||||
/** Levinson–Durbin recursion over {@link #autoCorr}, writing {@link #reflection}. */
|
||||
private void computeInitialInverseFilterCoefficients() {
|
||||
float error = autoCorr[0];
|
||||
for (int i = 0; i < NUM_LPC_COEFFICIENTS - 1; i++) {
|
||||
float reflectionCoeff = 0;
|
||||
for (int j = 0; j < i; j++) {
|
||||
reflectionCoeff += reflection[j] * autoCorr[i - j];
|
||||
}
|
||||
reflectionCoeff += autoCorr[i + 1];
|
||||
|
||||
// Avoid dividing by numbers close to zero.
|
||||
final float minErrorMagnitude = 1e-6f;
|
||||
if (Math.abs(error) < minErrorMagnitude) {
|
||||
error = Math.copySign(minErrorMagnitude, error);
|
||||
}
|
||||
reflectionCoeff /= -error;
|
||||
|
||||
reflection[i] = reflectionCoeff;
|
||||
for (int j = 0; j < ((i + 1) >> 1); j++) {
|
||||
float tmp1 = reflection[j];
|
||||
float tmp2 = reflection[i - 1 - j];
|
||||
reflection[j] = tmp1 + reflectionCoeff * tmp2;
|
||||
reflection[i - 1 - j] = tmp2 + reflectionCoeff * tmp1;
|
||||
}
|
||||
error -= reflectionCoeff * reflectionCoeff * error;
|
||||
if (error < 0.001f * autoCorr[0]) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the inverse filter: {@code y[i] = x[i] + sum_j lpc[j] * x[i - 1 - j]},
|
||||
* with the sum truncated near the start of the buffer.
|
||||
*/
|
||||
static void computeLpResidual(float[] lpcCoeffs, float[] x, float[] y) {
|
||||
for (int i = 0; i < x.length; i++) {
|
||||
float sum = x[i];
|
||||
int taps = Math.min(i, NUM_LPC_COEFFICIENTS);
|
||||
for (int j = 0; j < taps; j++) {
|
||||
sum += lpcCoeffs[j] * x[i - 1 - j];
|
||||
}
|
||||
y[i] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,329 @@
|
||||
package com.ts3client.audio.vad;
|
||||
|
||||
import static com.ts3client.audio.vad.VadConstants.BUF_SIZE_12K;
|
||||
import static com.ts3client.audio.vad.VadConstants.FRAME_20MS_12K;
|
||||
import static com.ts3client.audio.vad.VadConstants.FRAME_20MS_24K;
|
||||
import static com.ts3client.audio.vad.VadConstants.INITIAL_NUM_LAGS_24K;
|
||||
import static com.ts3client.audio.vad.VadConstants.MAX_PITCH_24K;
|
||||
import static com.ts3client.audio.vad.VadConstants.MAX_PITCH_48K;
|
||||
import static com.ts3client.audio.vad.VadConstants.MIN_PITCH_24K;
|
||||
import static com.ts3client.audio.vad.VadConstants.MIN_PITCH_48K;
|
||||
import static com.ts3client.audio.vad.VadConstants.NUM_LAGS_12K;
|
||||
import static com.ts3client.audio.vad.VadConstants.REFINE_NUM_LAGS_24K;
|
||||
|
||||
/**
|
||||
* Pitch period estimation for {@code rnn_vad}, ported from WebRTC's
|
||||
* {@code rnn_vad/pitch_search.cc} and {@code pitch_search_internal.cc}.
|
||||
*
|
||||
* <p>Three stages: a coarse search over the 12 kHz decimated LP residual picks two
|
||||
* candidates, those are refined against the 24 kHz buffer, and finally sub-harmonics
|
||||
* of the winner are tested so that an octave error can be corrected. The result is a
|
||||
* period expressed at 48 kHz, which is what the network was trained on.
|
||||
*
|
||||
* <p>Lags are stored inverted: index 0 corresponds to the maximum pitch period.
|
||||
*/
|
||||
final class PitchEstimator {
|
||||
|
||||
/**
|
||||
* For each divisor k in [2, 15], the multiplier n such that n*T/k is a sub-harmonic
|
||||
* worth checking alongside T/k, chosen so no multiple is visited twice.
|
||||
*/
|
||||
private static final int[] SUB_HARMONIC_MULTIPLIERS =
|
||||
{3, 2, 3, 2, 5, 2, 3, 2, 3, 2, 5, 2, 3, 2};
|
||||
|
||||
/** Thresholds on the initial period, computed as {@code [5*k*k for k in range(16)]}. */
|
||||
private static final int[] INITIAL_PITCH_PERIOD_THRESHOLDS =
|
||||
{20, 45, 80, 125, 180, 245, 320, 405, 500, 605, 720, 845, 980, 1125};
|
||||
|
||||
/** Number of pitch periods analysed either side of a candidate during refinement. */
|
||||
private static final int PITCH_NEIGHBORHOOD_RADIUS = 2;
|
||||
|
||||
private final float[] pitchBuffer12k = new float[BUF_SIZE_12K];
|
||||
private final float[] autoCorrelation12k = new float[NUM_LAGS_12K];
|
||||
private final float[] yEnergy24k = new float[REFINE_NUM_LAGS_24K];
|
||||
private final float[] autoCorrelation24k = new float[INITIAL_NUM_LAGS_24K];
|
||||
private final int[] invertedLagsIndex = new int[2 * (2 * PITCH_NEIGHBORHOOD_RADIUS + 1)];
|
||||
private int invertedLagsCount;
|
||||
|
||||
// Pitch tracking: the previous estimate biases the sub-harmonic decision.
|
||||
private int lastPeriod48k;
|
||||
private float lastStrength;
|
||||
|
||||
void reset() {
|
||||
lastPeriod48k = 0;
|
||||
lastStrength = 0;
|
||||
}
|
||||
|
||||
/** Returns the pitch period at 48 kHz for the given 24 kHz LP residual buffer. */
|
||||
int estimate(float[] pitchBuffer) {
|
||||
// Coarse search at 12 kHz. WebRTC decimates without an anti-aliasing filter here;
|
||||
// the refinement stage below is what recovers the accuracy.
|
||||
for (int i = 0; i < BUF_SIZE_12K; i++) {
|
||||
pitchBuffer12k[i] = pitchBuffer[2 * i];
|
||||
}
|
||||
computeAutoCorrelation12k();
|
||||
long candidates = computePitchPeriod12kHz();
|
||||
int best = (int) (candidates >> 32) * 2;
|
||||
int secondBest = (int) candidates * 2;
|
||||
|
||||
computeSlidingFrameSquareEnergies24k(pitchBuffer);
|
||||
int pitchLag48k = computePitchPeriod48kHz(pitchBuffer, best, secondBest);
|
||||
computeExtendedPitchPeriod48kHz(pitchBuffer, MAX_PITCH_48K - pitchLag48k);
|
||||
return lastPeriod48k;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@code autoCorrelation12k[lag]} is the correlation of the most recent 20 ms with the
|
||||
* frame starting {@code lag} samples into the buffer.
|
||||
*/
|
||||
private void computeAutoCorrelation12k() {
|
||||
final int refOffset = BUF_SIZE_12K - FRAME_20MS_12K;
|
||||
for (int lag = 0; lag < NUM_LAGS_12K; lag++) {
|
||||
float sum = 0;
|
||||
for (int i = 0; i < FRAME_20MS_12K; i++) {
|
||||
sum += pitchBuffer12k[refOffset + i] * pitchBuffer12k[lag + i];
|
||||
}
|
||||
autoCorrelation12k[lag] = sum;
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns the best and second-best inverted lags packed into a long. */
|
||||
private long computePitchPeriod12kHz() {
|
||||
float denominator = 1.0f;
|
||||
for (int i = 0; i < FRAME_20MS_12K + 1; i++) {
|
||||
denominator += pitchBuffer12k[i] * pitchBuffer12k[i];
|
||||
}
|
||||
|
||||
int bestLag = 0;
|
||||
float bestNumerator = -1.0f;
|
||||
float bestDenominator = 0.0f;
|
||||
int secondLag = 1;
|
||||
float secondNumerator = -1.0f;
|
||||
float secondDenominator = 0.0f;
|
||||
|
||||
for (int invertedLag = 0; invertedLag < NUM_LAGS_12K; invertedLag++) {
|
||||
// A pitch candidate must have positive correlation.
|
||||
if (autoCorrelation12k[invertedLag] > 0.0f) {
|
||||
float numerator = autoCorrelation12k[invertedLag] * autoCorrelation12k[invertedLag];
|
||||
// Compare numerator/denominator ratios without dividing.
|
||||
if (numerator * secondDenominator > secondNumerator * denominator) {
|
||||
if (numerator * bestDenominator > bestNumerator * denominator) {
|
||||
secondLag = bestLag;
|
||||
secondNumerator = bestNumerator;
|
||||
secondDenominator = bestDenominator;
|
||||
bestLag = invertedLag;
|
||||
bestNumerator = numerator;
|
||||
bestDenominator = denominator;
|
||||
} else {
|
||||
secondLag = invertedLag;
|
||||
secondNumerator = numerator;
|
||||
secondDenominator = denominator;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Slide the energy window to the next inverted lag.
|
||||
float yOld = pitchBuffer12k[invertedLag];
|
||||
float yNew = pitchBuffer12k[invertedLag + FRAME_20MS_12K];
|
||||
denominator -= yOld * yOld;
|
||||
denominator += yNew * yNew;
|
||||
denominator = Math.max(0.0f, denominator);
|
||||
}
|
||||
return ((long) bestLag << 32) | (secondLag & 0xffffffffL);
|
||||
}
|
||||
|
||||
/** Energy of the sliding 20 ms frame for every inverted lag. */
|
||||
private void computeSlidingFrameSquareEnergies24k(float[] pitchBuffer) {
|
||||
float yy = 0;
|
||||
for (int i = 0; i < FRAME_20MS_24K; i++) {
|
||||
yy += pitchBuffer[i] * pitchBuffer[i];
|
||||
}
|
||||
yEnergy24k[0] = yy;
|
||||
for (int invertedLag = 0; invertedLag < MAX_PITCH_24K; invertedLag++) {
|
||||
yy -= pitchBuffer[invertedLag] * pitchBuffer[invertedLag];
|
||||
yy += pitchBuffer[invertedLag + FRAME_20MS_24K] * pitchBuffer[invertedLag + FRAME_20MS_24K];
|
||||
yy = Math.max(1.0f, yy);
|
||||
yEnergy24k[invertedLag + 1] = yy;
|
||||
}
|
||||
}
|
||||
|
||||
/** Correlation of the most recent 20 ms with the frame at {@code invertedLag}. */
|
||||
private static float autoCorrelation(int invertedLag, float[] pitchBuffer) {
|
||||
float sum = 0;
|
||||
for (int i = 0; i < FRAME_20MS_24K; i++) {
|
||||
sum += pitchBuffer[MAX_PITCH_24K + i] * pitchBuffer[invertedLag + i];
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recomputes the auto-correlation around both candidates at 24 kHz and returns the
|
||||
* strongest inverted lag, pseudo-interpolated to 48 kHz.
|
||||
*/
|
||||
private int computePitchPeriod48kHz(float[] pitchBuffer, int bestCandidate, int secondCandidate) {
|
||||
invertedLagsCount = 0;
|
||||
// Order the two neighbourhoods so the first precedes the second.
|
||||
boolean swap = bestCandidate > secondCandidate;
|
||||
int firstCenter = swap ? secondCandidate : bestCandidate;
|
||||
int secondCenter = swap ? bestCandidate : secondCandidate;
|
||||
int r1min = Math.max(firstCenter - PITCH_NEIGHBORHOOD_RADIUS, 0);
|
||||
int r1max = Math.min(firstCenter + PITCH_NEIGHBORHOOD_RADIUS, INITIAL_NUM_LAGS_24K - 1);
|
||||
int r2min = Math.max(secondCenter - PITCH_NEIGHBORHOOD_RADIUS, 0);
|
||||
int r2max = Math.min(secondCenter + PITCH_NEIGHBORHOOD_RADIUS, INITIAL_NUM_LAGS_24K - 1);
|
||||
|
||||
if (r1max + 1 >= r2min) { // Overlapping or adjacent ranges.
|
||||
computeAutoCorrelationRange(pitchBuffer, r1min, r2max);
|
||||
} else {
|
||||
computeAutoCorrelationRange(pitchBuffer, r1min, r1max);
|
||||
computeAutoCorrelationRange(pitchBuffer, r2min, r2max);
|
||||
}
|
||||
|
||||
int bestInvertedLag = 0;
|
||||
float bestNumerator = -1.0f;
|
||||
float bestDenominator = 0.0f;
|
||||
for (int i = 0; i < invertedLagsCount; i++) {
|
||||
int invertedLag = invertedLagsIndex[i];
|
||||
if (autoCorrelation24k[invertedLag] > 0.0f) {
|
||||
float numerator = autoCorrelation24k[invertedLag] * autoCorrelation24k[invertedLag];
|
||||
float denominator = yEnergy24k[invertedLag];
|
||||
if (numerator * bestDenominator > bestNumerator * denominator) {
|
||||
bestInvertedLag = invertedLag;
|
||||
bestNumerator = numerator;
|
||||
bestDenominator = denominator;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (bestInvertedLag == 0 || bestInvertedLag >= INITIAL_NUM_LAGS_24K - 1) {
|
||||
return bestInvertedLag * 2; // Cannot interpolate at the boundaries.
|
||||
}
|
||||
int offset = pseudoInterpolationOffset(
|
||||
autoCorrelation24k[bestInvertedLag + 1],
|
||||
autoCorrelation24k[bestInvertedLag],
|
||||
autoCorrelation24k[bestInvertedLag - 1]);
|
||||
return 2 * bestInvertedLag + offset;
|
||||
}
|
||||
|
||||
private void computeAutoCorrelationRange(float[] pitchBuffer, int min, int max) {
|
||||
// The pseudo-interpolation reads one lag either side, so zero those rather than
|
||||
// clearing the whole array.
|
||||
if (min > 0) {
|
||||
autoCorrelation24k[min - 1] = 0.0f;
|
||||
}
|
||||
if (max < INITIAL_NUM_LAGS_24K - 1) {
|
||||
autoCorrelation24k[max + 1] = 0.0f;
|
||||
}
|
||||
for (int invertedLag = min; invertedLag <= max; invertedLag++) {
|
||||
autoCorrelation24k[invertedLag] = autoCorrelation(invertedLag, pitchBuffer);
|
||||
invertedLagsIndex[invertedLagsCount++] = invertedLag;
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns a lag correction in {-1, 0, +1} based on the neighbouring correlations. */
|
||||
private static int pseudoInterpolationOffset(float prev, float curr, float next) {
|
||||
if ((next - prev) > 0.7f * (curr - prev)) {
|
||||
return 1;
|
||||
} else if ((prev - next) > 0.7f * (curr - next)) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Refines {@code lag} (24 kHz) to a 48 kHz lag using pseudo-interpolation. */
|
||||
private static int pseudoInterpolationLagPitchBuf(int lag, float[] pitchBuffer) {
|
||||
int offset = 0;
|
||||
if (lag > 0 && lag < MAX_PITCH_24K) {
|
||||
int invertedLag = MAX_PITCH_24K - lag;
|
||||
offset = pseudoInterpolationOffset(
|
||||
autoCorrelation(invertedLag + 1, pitchBuffer),
|
||||
autoCorrelation(invertedLag, pitchBuffer),
|
||||
autoCorrelation(invertedLag - 1, pitchBuffer));
|
||||
}
|
||||
return 2 * lag + offset;
|
||||
}
|
||||
|
||||
/** Same as {@code round(multiplier * pitchPeriod / divisor)}. */
|
||||
private static int alternativePitchPeriod(int pitchPeriod, int multiplier, int divisor) {
|
||||
return (2 * multiplier * pitchPeriod + divisor) / (2 * divisor);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests sub-harmonics of the refined period and keeps the strongest, then stores the
|
||||
* winner in {@link #lastPeriod48k}/{@link #lastStrength}.
|
||||
*/
|
||||
private void computeExtendedPitchPeriod48kHz(float[] pitchBuffer, int initialPeriod48k) {
|
||||
final float xEnergy = yEnergy24k[MAX_PITCH_24K];
|
||||
|
||||
int bestPeriod = Math.min(initialPeriod48k / 2, MAX_PITCH_24K - 1);
|
||||
float bestXy = autoCorrelation(MAX_PITCH_24K - bestPeriod, pitchBuffer);
|
||||
float bestYEnergy = yEnergy24k[MAX_PITCH_24K - bestPeriod];
|
||||
float bestStrength = pitchStrength(bestXy, bestYEnergy, xEnergy);
|
||||
|
||||
final int initialPeriod = bestPeriod;
|
||||
final float initialStrength = bestStrength;
|
||||
// The tracked pitch is kept at 48 kHz but compared at 24 kHz.
|
||||
final int lastPeriod = lastPeriod48k / 2;
|
||||
|
||||
// Largest divisor for which the alternative period is still above the minimum.
|
||||
int maxPeriodDivisor = (2 * initialPeriod) / (2 * MIN_PITCH_24K - 1);
|
||||
for (int periodDivisor = 2; periodDivisor <= maxPeriodDivisor; periodDivisor++) {
|
||||
int alternativePeriod = alternativePitchPeriod(initialPeriod, 1, periodDivisor);
|
||||
int dualPeriod = alternativePitchPeriod(
|
||||
initialPeriod, SUB_HARMONIC_MULTIPLIERS[periodDivisor - 2], periodDivisor);
|
||||
// Special case: for a divisor of 2 the sub-harmonic can exceed the pitch range.
|
||||
if (periodDivisor == 2 && dualPeriod > MAX_PITCH_24K) {
|
||||
dualPeriod = initialPeriod;
|
||||
}
|
||||
|
||||
// Score the candidate together with its sub-harmonic.
|
||||
float xyPrimary = autoCorrelation(MAX_PITCH_24K - alternativePeriod, pitchBuffer);
|
||||
float xySecondary = autoCorrelation(MAX_PITCH_24K - dualPeriod, pitchBuffer);
|
||||
float xy = 0.5f * (xyPrimary + xySecondary);
|
||||
float yy = 0.5f * (yEnergy24k[MAX_PITCH_24K - alternativePeriod]
|
||||
+ yEnergy24k[MAX_PITCH_24K - dualPeriod]);
|
||||
float alternativeStrength = pitchStrength(xy, yy, xEnergy);
|
||||
|
||||
if (isAlternativeStronger(lastPeriod, initialPeriod, initialStrength,
|
||||
alternativePeriod, alternativeStrength, periodDivisor)) {
|
||||
bestPeriod = alternativePeriod;
|
||||
bestStrength = alternativeStrength;
|
||||
bestXy = xy;
|
||||
bestYEnergy = yy;
|
||||
}
|
||||
}
|
||||
|
||||
bestXy = Math.max(0.0f, bestXy);
|
||||
float finalStrength = (bestYEnergy <= bestXy) ? 1.0f : bestXy / (bestYEnergy + 1.0f);
|
||||
finalStrength = Math.min(bestStrength, finalStrength);
|
||||
|
||||
lastPeriod48k = Math.max(MIN_PITCH_48K, pseudoInterpolationLagPitchBuf(bestPeriod, pitchBuffer));
|
||||
lastStrength = finalStrength;
|
||||
}
|
||||
|
||||
private static float pitchStrength(float xy, float yEnergy, float xEnergy) {
|
||||
return xy / (float) Math.sqrt(1.0f + xEnergy * yEnergy);
|
||||
}
|
||||
|
||||
private boolean isAlternativeStronger(int lastPeriod, int initialPeriod, float initialStrength,
|
||||
int alternativePeriod, float alternativeStrength,
|
||||
int periodDivisor) {
|
||||
// Pitch tracking: a candidate close to the previous estimate clears a lower bar.
|
||||
float lowerThresholdTerm = 0.0f;
|
||||
int distance = Math.abs(alternativePeriod - lastPeriod);
|
||||
if (distance <= 1) {
|
||||
lowerThresholdTerm = lastStrength;
|
||||
} else if (distance == 2
|
||||
&& initialPeriod > INITIAL_PITCH_PERIOD_THRESHOLDS[periodDivisor - 2]) {
|
||||
lowerThresholdTerm = 0.5f * lastStrength;
|
||||
}
|
||||
|
||||
// Higher frequencies get a stricter threshold: short-term correlations bias
|
||||
// the search towards them and cause false positives.
|
||||
float threshold = Math.max(0.3f, 0.7f * initialStrength - lowerThresholdTerm);
|
||||
if (alternativePeriod < 3 * MIN_PITCH_24K) {
|
||||
threshold = Math.max(0.4f, 0.85f * initialStrength - lowerThresholdTerm);
|
||||
} else if (alternativePeriod < 2 * MIN_PITCH_24K) {
|
||||
threshold = Math.max(0.5f, 0.9f * initialStrength - lowerThresholdTerm);
|
||||
}
|
||||
return alternativeStrength > threshold;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
package com.ts3client.audio.vad;
|
||||
|
||||
/**
|
||||
* Real-input FFT for the {@code rnn_vad} spectral analysis.
|
||||
*
|
||||
* <p>The transform length is 480 (a 20 ms frame at 24 kHz), which is not a
|
||||
* power of two, so the shared radix-2 {@code Fft} cannot be used. This is a
|
||||
* mixed-radix Cooley–Tukey complex FFT (radices 2, 3, 4 and 5, with a generic
|
||||
* DFT for any other factor) wrapped in the standard real-input trick: an
|
||||
* {@code n}-point real transform is computed as an {@code n/2}-point complex one
|
||||
* followed by a split step.
|
||||
*
|
||||
* <p>Output uses the packed layout WebRTC's PFFFT produces, which the band
|
||||
* correlator indexes directly:
|
||||
* <pre>
|
||||
* out[0] = Re(X[0]) (DC)
|
||||
* out[1] = Re(X[n/2]) (Nyquist)
|
||||
* out[2k] = Re(X[k])
|
||||
* out[2k+1] = Im(X[k]) for k in [1, n/2)
|
||||
* </pre>
|
||||
*/
|
||||
final class RealFft {
|
||||
|
||||
private final int size;
|
||||
private final int half;
|
||||
|
||||
// Twiddles for the complex sub-transform and for the real split step.
|
||||
private final float[] halfRe;
|
||||
private final float[] halfIm;
|
||||
private final float[] splitCos;
|
||||
private final float[] splitSin;
|
||||
|
||||
private final ComplexFft complexFft;
|
||||
|
||||
RealFft(int size) {
|
||||
if (size < 2 || (size & 1) != 0) {
|
||||
throw new IllegalArgumentException("Real FFT length must be even: " + size);
|
||||
}
|
||||
this.size = size;
|
||||
this.half = size / 2;
|
||||
this.halfRe = new float[half];
|
||||
this.halfIm = new float[half];
|
||||
this.complexFft = new ComplexFft(half);
|
||||
|
||||
this.splitCos = new float[half / 2 + 1];
|
||||
this.splitSin = new float[half / 2 + 1];
|
||||
for (int k = 0; k <= half / 2; k++) {
|
||||
double angle = -2.0 * Math.PI * k / size;
|
||||
splitCos[k] = (float) Math.cos(angle);
|
||||
splitSin[k] = (float) Math.sin(angle);
|
||||
}
|
||||
}
|
||||
|
||||
int size() {
|
||||
return size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Transforms {@code in} ({@link #size} real samples) into {@code out}
|
||||
* ({@link #size} floats in the packed layout described above).
|
||||
*/
|
||||
void forward(float[] in, float[] out) {
|
||||
// Pack the real signal as half/2 complex samples: z[i] = x[2i] + j*x[2i+1].
|
||||
for (int i = 0; i < half; i++) {
|
||||
halfRe[i] = in[2 * i];
|
||||
halfIm[i] = in[2 * i + 1];
|
||||
}
|
||||
complexFft.forward(halfRe, halfIm);
|
||||
|
||||
// Split Z into the transforms of the even and odd sample sequences, then
|
||||
// recombine them into the spectrum of the original real signal.
|
||||
out[0] = halfRe[0] + halfIm[0];
|
||||
out[1] = halfRe[0] - halfIm[0];
|
||||
for (int k = 1; k <= half / 2; k++) {
|
||||
int mirror = half - k;
|
||||
|
||||
// Even part: hermitian-symmetric average; odd part: the difference.
|
||||
float evenRe = 0.5f * (halfRe[k] + halfRe[mirror]);
|
||||
float evenIm = 0.5f * (halfIm[k] - halfIm[mirror]);
|
||||
float oddRe = 0.5f * (halfIm[k] + halfIm[mirror]);
|
||||
float oddIm = -0.5f * (halfRe[k] - halfRe[mirror]);
|
||||
|
||||
float c = splitCos[k];
|
||||
float s = splitSin[k];
|
||||
float twRe = oddRe * c - oddIm * s;
|
||||
float twIm = oddRe * s + oddIm * c;
|
||||
|
||||
float re = evenRe + twRe;
|
||||
float im = evenIm + twIm;
|
||||
out[2 * k] = re;
|
||||
out[2 * k + 1] = im;
|
||||
|
||||
if (mirror != k) {
|
||||
// X[n/2 - k] is the conjugate-mirrored companion of X[k].
|
||||
out[2 * mirror] = evenRe - twRe;
|
||||
out[2 * mirror + 1] = -(evenIm - twIm);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mixed-radix complex FFT, decimation in time.
|
||||
*
|
||||
* <p>A transform of length {@code n = r * m} is computed by transforming the {@code r}
|
||||
* interleaved subsequences of length {@code m} and combining them:
|
||||
* {@code X[k1*m + k2] = sum_j (W_n^(j*k2) * X_j[k2]) * W_r^(j*k1)}.
|
||||
*
|
||||
* <p>The recursion is expressed directly rather than as an iterative digit-reversed
|
||||
* loop, which keeps it correct for any factorisation. Scratch buffers are allocated
|
||||
* once per recursion level so no transform allocates.
|
||||
*/
|
||||
static final class ComplexFft {
|
||||
|
||||
private final int n;
|
||||
private final float[] cosTable;
|
||||
private final float[] sinTable;
|
||||
private final float[][] scratchRe;
|
||||
private final float[][] scratchIm;
|
||||
private final float[] dftRe;
|
||||
private final float[] dftIm;
|
||||
|
||||
ComplexFft(int n) {
|
||||
this.n = n;
|
||||
this.cosTable = new float[n];
|
||||
this.sinTable = new float[n];
|
||||
for (int i = 0; i < n; i++) {
|
||||
double angle = -2.0 * Math.PI * i / n;
|
||||
cosTable[i] = (float) Math.cos(angle);
|
||||
sinTable[i] = (float) Math.sin(angle);
|
||||
}
|
||||
int depth = 0;
|
||||
int maxRadix = 1;
|
||||
for (int rest = n; rest > 1; depth++) {
|
||||
int r = smallestFactor(rest);
|
||||
maxRadix = Math.max(maxRadix, r);
|
||||
rest /= r;
|
||||
}
|
||||
this.scratchRe = new float[depth + 1][n];
|
||||
this.scratchIm = new float[depth + 1][n];
|
||||
this.dftRe = new float[maxRadix];
|
||||
this.dftIm = new float[maxRadix];
|
||||
}
|
||||
|
||||
/** Transforms {@code re}/{@code im} of length {@code n} in place. */
|
||||
void forward(float[] re, float[] im) {
|
||||
transform(re, im, 0, 1, n, scratchRe[0], scratchIm[0], 0, 1);
|
||||
System.arraycopy(scratchRe[0], 0, re, 0, n);
|
||||
System.arraycopy(scratchIm[0], 0, im, 0, n);
|
||||
}
|
||||
|
||||
/**
|
||||
* Transforms the {@code len} samples starting at {@code inOff} with the given
|
||||
* {@code stride}, writing the spectrum to {@code outRe}/{@code outIm} at
|
||||
* {@code outOff}.
|
||||
*/
|
||||
private void transform(float[] inRe, float[] inIm, int inOff, int stride, int len,
|
||||
float[] outRe, float[] outIm, int outOff, int depth) {
|
||||
if (len == 1) {
|
||||
outRe[outOff] = inRe[inOff];
|
||||
outIm[outOff] = inIm[inOff];
|
||||
return;
|
||||
}
|
||||
int r = smallestFactor(len);
|
||||
int m = len / r;
|
||||
|
||||
// Sub-transforms of the r interleaved subsequences, laid out end to end.
|
||||
float[] subRe = scratchRe[depth];
|
||||
float[] subIm = scratchIm[depth];
|
||||
for (int j = 0; j < r; j++) {
|
||||
transform(inRe, inIm, inOff + j * stride, stride * r, m,
|
||||
subRe, subIm, outOff + j * m, depth + 1);
|
||||
}
|
||||
|
||||
// Twiddle step for this transform size, expressed in top-level table units.
|
||||
int unit = n / len;
|
||||
for (int k2 = 0; k2 < m; k2++) {
|
||||
for (int j = 0; j < r; j++) {
|
||||
int idx = outOff + j * m + k2;
|
||||
int t = (j * k2 * unit) % n;
|
||||
float c = cosTable[t];
|
||||
float s = sinTable[t];
|
||||
dftRe[j] = subRe[idx] * c - subIm[idx] * s;
|
||||
dftIm[j] = subRe[idx] * s + subIm[idx] * c;
|
||||
}
|
||||
combine(outRe, outIm, outOff + k2, m, r);
|
||||
}
|
||||
}
|
||||
|
||||
/** Length-{@code r} DFT of {@code dftRe}/{@code dftIm}, written with stride {@code m}. */
|
||||
private void combine(float[] outRe, float[] outIm, int outOff, int m, int r) {
|
||||
switch (r) {
|
||||
case 2 -> {
|
||||
outRe[outOff] = dftRe[0] + dftRe[1];
|
||||
outIm[outOff] = dftIm[0] + dftIm[1];
|
||||
outRe[outOff + m] = dftRe[0] - dftRe[1];
|
||||
outIm[outOff + m] = dftIm[0] - dftIm[1];
|
||||
}
|
||||
default -> {
|
||||
int unit = n / r;
|
||||
for (int k1 = 0; k1 < r; k1++) {
|
||||
float sumRe = 0;
|
||||
float sumIm = 0;
|
||||
for (int j = 0; j < r; j++) {
|
||||
int t = (j * k1 % r) * unit;
|
||||
float c = cosTable[t];
|
||||
float s = sinTable[t];
|
||||
sumRe += dftRe[j] * c - dftIm[j] * s;
|
||||
sumIm += dftRe[j] * s + dftIm[j] * c;
|
||||
}
|
||||
outRe[outOff + k1 * m] = sumRe;
|
||||
outIm[outOff + k1 * m] = sumIm;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static int smallestFactor(int v) {
|
||||
for (int f = 2; f * f <= v; f++) {
|
||||
if (v % f == 0) return f;
|
||||
}
|
||||
return v;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package com.ts3client.audio.vad;
|
||||
|
||||
/**
|
||||
* The {@code rnn_vad} network: a 42→24 dense layer, a 24-unit GRU and a 24→1
|
||||
* dense output. Ported from WebRTC's {@code rnn_vad/rnn.cc}, {@code rnn_fc.cc} and
|
||||
* {@code rnn_gru.cc}, using the activation approximations from RNNoise.
|
||||
*
|
||||
* <p>The GRU carries state between frames, so the caller must feed frames in order and
|
||||
* {@link #reset()} on a discontinuity. Silence resets the state, matching WebRTC.
|
||||
*/
|
||||
final class RnnNetwork {
|
||||
|
||||
private static final int INPUT_SIZE = 42;
|
||||
private static final int FC1_SIZE = 24;
|
||||
private static final int GRU_SIZE = 24;
|
||||
private static final int NUM_GATES = 3; // update, reset, output
|
||||
|
||||
private final float[] fc1Weights; // [output][input]
|
||||
private final float[] fc1Bias;
|
||||
private final float[] fc2Weights;
|
||||
private final float[] fc2Bias;
|
||||
|
||||
private final float[] gruWeights; // [gate][output][input]
|
||||
private final float[] gruRecurrentWeights; // [gate][output][output]
|
||||
private final float[] gruBias; // [gate][output]
|
||||
|
||||
private final float[] fc1Output = new float[FC1_SIZE];
|
||||
private final float[] state = new float[GRU_SIZE];
|
||||
private final float[] update = new float[GRU_SIZE];
|
||||
private final float[] resetGate = new float[GRU_SIZE];
|
||||
private final float[] resetTimesState = new float[GRU_SIZE];
|
||||
|
||||
RnnNetwork() {
|
||||
fc1Weights = transposeAndScale(RnnVadWeights.INPUT_DENSE_WEIGHTS, INPUT_SIZE, FC1_SIZE);
|
||||
fc1Bias = scale(RnnVadWeights.INPUT_DENSE_BIAS);
|
||||
// The output layer is 24x1, so its stored layout already matches.
|
||||
fc2Weights = scale(RnnVadWeights.OUTPUT_DENSE_WEIGHTS);
|
||||
fc2Bias = scale(RnnVadWeights.OUTPUT_DENSE_BIAS);
|
||||
gruWeights = transposeGruTensor(RnnVadWeights.HIDDEN_GRU_WEIGHTS, FC1_SIZE, GRU_SIZE);
|
||||
gruRecurrentWeights =
|
||||
transposeGruTensor(RnnVadWeights.HIDDEN_GRU_RECURRENT_WEIGHTS, GRU_SIZE, GRU_SIZE);
|
||||
gruBias = transposeGruTensor(RnnVadWeights.HIDDEN_GRU_BIAS, 1, GRU_SIZE);
|
||||
}
|
||||
|
||||
void reset() {
|
||||
java.util.Arrays.fill(state, 0.0f);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the speech probability for a feature vector. When {@code silence} is set the
|
||||
* state is cleared and zero is returned without running the network.
|
||||
*/
|
||||
float computeVadProbability(float[] featureVector, boolean silence) {
|
||||
if (silence) {
|
||||
reset();
|
||||
return 0.0f;
|
||||
}
|
||||
for (int o = 0; o < FC1_SIZE; o++) {
|
||||
float x = fc1Bias[o] + dot(featureVector, fc1Weights, o * INPUT_SIZE, INPUT_SIZE);
|
||||
fc1Output[o] = tansigApproximated(x);
|
||||
}
|
||||
computeGruOutput(fc1Output);
|
||||
float x = fc2Bias[0] + dot(state, fc2Weights, 0, GRU_SIZE);
|
||||
return sigmoidApproximated(x);
|
||||
}
|
||||
|
||||
private void computeGruOutput(float[] input) {
|
||||
final int strideWeights = FC1_SIZE * GRU_SIZE;
|
||||
final int strideRecurrent = GRU_SIZE * GRU_SIZE;
|
||||
|
||||
// u = sigmoid(W_u.i + R_u.s + b_u)
|
||||
computeGate(input, 0, 0, 0, update);
|
||||
// r = sigmoid(W_r.i + R_r.s + b_r)
|
||||
computeGate(input, GRU_SIZE, strideWeights, strideRecurrent, resetGate);
|
||||
|
||||
// s' = u.s + (1 - u).ReLU(W_s.i + R_s.(s.r) + b_s)
|
||||
for (int o = 0; o < GRU_SIZE; o++) {
|
||||
resetTimesState[o] = state[o] * resetGate[o];
|
||||
}
|
||||
for (int o = 0; o < GRU_SIZE; o++) {
|
||||
float x = gruBias[2 * GRU_SIZE + o];
|
||||
x += dot(input, gruWeights, 2 * strideWeights + o * FC1_SIZE, FC1_SIZE);
|
||||
x += dot(resetTimesState, gruRecurrentWeights,
|
||||
2 * strideRecurrent + o * GRU_SIZE, GRU_SIZE);
|
||||
state[o] = update[o] * state[o] + (1.0f - update[o]) * Math.max(0.0f, x);
|
||||
}
|
||||
}
|
||||
|
||||
private void computeGate(float[] input, int biasOffset, int weightsOffset,
|
||||
int recurrentOffset, float[] gate) {
|
||||
for (int o = 0; o < GRU_SIZE; o++) {
|
||||
float x = gruBias[biasOffset + o];
|
||||
x += dot(input, gruWeights, weightsOffset + o * FC1_SIZE, FC1_SIZE);
|
||||
x += dot(state, gruRecurrentWeights, recurrentOffset + o * GRU_SIZE, GRU_SIZE);
|
||||
gate[o] = sigmoidApproximated(x);
|
||||
}
|
||||
}
|
||||
|
||||
private static float dot(float[] a, float[] weights, int weightsOffset, int len) {
|
||||
float sum = 0;
|
||||
for (int i = 0; i < len; i++) {
|
||||
sum += a[i] * weights[weightsOffset + i];
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// ---- weight layout ----
|
||||
|
||||
private static float[] scale(byte[] src) {
|
||||
float[] out = new float[src.length];
|
||||
for (int i = 0; i < src.length; i++) {
|
||||
out[i] = RnnVadWeights.SCALE * src[i];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Stored as {@code [input][output]}; the dot products want {@code [output][input]}. */
|
||||
private static float[] transposeAndScale(byte[] src, int inputSize, int outputSize) {
|
||||
float[] out = new float[src.length];
|
||||
for (int o = 0; o < outputSize; o++) {
|
||||
for (int i = 0; i < inputSize; i++) {
|
||||
out[o * inputSize + i] = RnnVadWeights.SCALE * src[i * outputSize + o];
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* GRU tensors are stored as {@code [n][gate][output]} and are wanted as
|
||||
* {@code [gate][output][n]}, where {@code n} is the input size (1 for the bias).
|
||||
*/
|
||||
private static float[] transposeGruTensor(byte[] src, int n, int outputSize) {
|
||||
float[] out = new float[src.length];
|
||||
int strideSrc = NUM_GATES * outputSize;
|
||||
int strideDst = n * outputSize;
|
||||
for (int g = 0; g < NUM_GATES; g++) {
|
||||
for (int o = 0; o < outputSize; o++) {
|
||||
for (int i = 0; i < n; i++) {
|
||||
out[g * strideDst + o * n + i] =
|
||||
RnnVadWeights.SCALE * src[i * strideSrc + g * outputSize + o];
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---- activations (RNNoise approximations, bit-compatible with WebRTC) ----
|
||||
|
||||
/** tanh sampled every 0.04, copied verbatim from RNNoise's {@code rnn_activations.h}. */
|
||||
private static final float[] TANSIG_TABLE = {
|
||||
0.000000f, 0.039979f, 0.079830f, 0.119427f, 0.158649f, 0.197375f, 0.235496f, 0.272905f,
|
||||
0.309507f, 0.345214f, 0.379949f, 0.413644f, 0.446244f, 0.477700f, 0.507977f, 0.537050f,
|
||||
0.564900f, 0.591519f, 0.616909f, 0.641077f, 0.664037f, 0.685809f, 0.706419f, 0.725897f,
|
||||
0.744277f, 0.761594f, 0.777888f, 0.793199f, 0.807569f, 0.821040f, 0.833655f, 0.845456f,
|
||||
0.856485f, 0.866784f, 0.876393f, 0.885352f, 0.893698f, 0.901468f, 0.908698f, 0.915420f,
|
||||
0.921669f, 0.927473f, 0.932862f, 0.937863f, 0.942503f, 0.946806f, 0.950795f, 0.954492f,
|
||||
0.957917f, 0.961090f, 0.964028f, 0.966747f, 0.969265f, 0.971594f, 0.973749f, 0.975743f,
|
||||
0.977587f, 0.979293f, 0.980869f, 0.982327f, 0.983675f, 0.984921f, 0.986072f, 0.987136f,
|
||||
0.988119f, 0.989027f, 0.989867f, 0.990642f, 0.991359f, 0.992020f, 0.992631f, 0.993196f,
|
||||
0.993718f, 0.994199f, 0.994644f, 0.995055f, 0.995434f, 0.995784f, 0.996108f, 0.996407f,
|
||||
0.996682f, 0.996937f, 0.997172f, 0.997389f, 0.997590f, 0.997775f, 0.997946f, 0.998104f,
|
||||
0.998249f, 0.998384f, 0.998508f, 0.998623f, 0.998728f, 0.998826f, 0.998916f, 0.999000f,
|
||||
0.999076f, 0.999147f, 0.999213f, 0.999273f, 0.999329f, 0.999381f, 0.999428f, 0.999472f,
|
||||
0.999513f, 0.999550f, 0.999585f, 0.999617f, 0.999646f, 0.999673f, 0.999699f, 0.999722f,
|
||||
0.999743f, 0.999763f, 0.999781f, 0.999798f, 0.999813f, 0.999828f, 0.999841f, 0.999853f,
|
||||
0.999865f, 0.999875f, 0.999885f, 0.999893f, 0.999902f, 0.999909f, 0.999916f, 0.999923f,
|
||||
0.999929f, 0.999934f, 0.999939f, 0.999944f, 0.999948f, 0.999952f, 0.999956f, 0.999959f,
|
||||
0.999962f, 0.999965f, 0.999968f, 0.999970f, 0.999973f, 0.999975f, 0.999977f, 0.999978f,
|
||||
0.999980f, 0.999982f, 0.999983f, 0.999984f, 0.999986f, 0.999987f, 0.999988f, 0.999989f,
|
||||
0.999990f, 0.999990f, 0.999991f, 0.999992f, 0.999992f, 0.999993f, 0.999994f, 0.999994f,
|
||||
0.999994f, 0.999995f, 0.999995f, 0.999996f, 0.999996f, 0.999996f, 0.999997f, 0.999997f,
|
||||
0.999997f, 0.999997f, 0.999997f, 0.999998f, 0.999998f, 0.999998f, 0.999998f, 0.999998f,
|
||||
0.999998f, 0.999999f, 0.999999f, 0.999999f, 0.999999f, 0.999999f, 0.999999f, 0.999999f,
|
||||
0.999999f, 0.999999f, 0.999999f, 0.999999f, 0.999999f, 0.999999f, 1.000000f, 1.000000f,
|
||||
1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f, 1.000000f,
|
||||
1.000000f
|
||||
};
|
||||
|
||||
private static float tansigApproximated(float x) {
|
||||
// Tests are reversed to catch NaNs.
|
||||
if (!(x < 8.0f)) {
|
||||
return 1.0f;
|
||||
}
|
||||
if (!(x > -8.0f)) {
|
||||
return -1.0f;
|
||||
}
|
||||
float sign = 1.0f;
|
||||
if (x < 0.0f) {
|
||||
x = -x;
|
||||
sign = -1.0f;
|
||||
}
|
||||
int i = (int) Math.floor(0.5f + 25 * x);
|
||||
float y = TANSIG_TABLE[i];
|
||||
// Newton step back to x's own scale (undoing the table's factor of 25).
|
||||
x -= 0.04f * i;
|
||||
y = y + x * (1.0f - y * y) * (1.0f - y * x);
|
||||
return sign * y;
|
||||
}
|
||||
|
||||
private static float sigmoidApproximated(float x) {
|
||||
return 0.5f + 0.5f * tansigApproximated(0.5f * x);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package com.ts3client.audio.vad;
|
||||
|
||||
import com.ts3client.audio.SpeechProbabilityDetector;
|
||||
|
||||
/**
|
||||
* {@link SpeechProbabilityDetector} backed by {@link RnnVad}, adapting an arbitrary
|
||||
* capture rate and frame size to the detector's fixed 10 ms frames at 24 kHz.
|
||||
*
|
||||
* <p>Capture frames rarely divide into 10 ms chunks exactly, so leftover samples are
|
||||
* carried into the next call. The reported probability is that of the most recent complete
|
||||
* 10 ms frame; a capture frame long enough to span several of them advances the
|
||||
* detector several times, exactly as feeding it directly would.
|
||||
*/
|
||||
public final class RnnSpeechDetector implements SpeechProbabilityDetector {
|
||||
|
||||
/** Half-band FIR used when decimating; long enough to keep aliasing out of the passband. */
|
||||
private static final int FILTER_TAPS = 32;
|
||||
|
||||
private final RnnVad vad = new RnnVad();
|
||||
|
||||
private final int sampleRate;
|
||||
private final boolean decimateByTwo;
|
||||
private final float[] filterCoeffs;
|
||||
private final float[] filterState;
|
||||
private int filterPos;
|
||||
|
||||
/** Resampled samples not yet consumed as a whole 10 ms frame. */
|
||||
private final float[] pending = new float[RnnVad.FRAME_SIZE];
|
||||
private int pendingCount;
|
||||
|
||||
/** Fractional read position, for capture rates that are not a multiple of 24 kHz. */
|
||||
private double resamplePos;
|
||||
|
||||
public RnnSpeechDetector(int sampleRate) {
|
||||
this.sampleRate = sampleRate;
|
||||
this.decimateByTwo = sampleRate == 2 * RnnVad.SAMPLE_RATE;
|
||||
if (decimateByTwo) {
|
||||
this.filterCoeffs = lowPass(FILTER_TAPS, 0.25);
|
||||
this.filterState = new float[FILTER_TAPS];
|
||||
} else {
|
||||
this.filterCoeffs = null;
|
||||
this.filterState = null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public double process(float[] frame, int length) {
|
||||
if (sampleRate == RnnVad.SAMPLE_RATE) {
|
||||
feed(frame, 0, length, 1);
|
||||
} else if (decimateByTwo) {
|
||||
decimateAndFeed(frame, length);
|
||||
} else {
|
||||
resampleAndFeed(frame, length);
|
||||
}
|
||||
return vad.probability();
|
||||
}
|
||||
|
||||
@Override
|
||||
public double probability() {
|
||||
return vad.probability();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reset() {
|
||||
vad.reset();
|
||||
pendingCount = 0;
|
||||
resamplePos = 0;
|
||||
filterPos = 0;
|
||||
if (filterState != null) {
|
||||
java.util.Arrays.fill(filterState, 0.0f);
|
||||
}
|
||||
}
|
||||
|
||||
/** Appends samples (taking every {@code step}-th) and runs the detector per full frame. */
|
||||
private void feed(float[] src, int offset, int length, int step) {
|
||||
for (int i = offset; i < offset + length; i += step) {
|
||||
push(src[i]);
|
||||
}
|
||||
}
|
||||
|
||||
/** 48 kHz (the common case): low-pass, then keep every other sample. */
|
||||
private void decimateAndFeed(float[] frame, int length) {
|
||||
for (int i = 0; i < length; i++) {
|
||||
filterState[filterPos] = frame[i];
|
||||
filterPos = (filterPos + 1) % FILTER_TAPS;
|
||||
if ((i & 1) == 1) {
|
||||
float acc = 0;
|
||||
int idx = filterPos;
|
||||
for (int t = 0; t < FILTER_TAPS; t++) {
|
||||
acc += filterCoeffs[t] * filterState[idx];
|
||||
idx = (idx + 1) % FILTER_TAPS;
|
||||
}
|
||||
push(acc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Any other capture rate. Linear interpolation is enough here: the detector's own
|
||||
* analysis is far coarser than the resampling error.
|
||||
*/
|
||||
private void resampleAndFeed(float[] frame, int length) {
|
||||
double ratio = (double) sampleRate / RnnVad.SAMPLE_RATE;
|
||||
while (resamplePos < length) {
|
||||
int i = (int) resamplePos;
|
||||
double frac = resamplePos - i;
|
||||
float a = frame[i];
|
||||
float b = frame[Math.min(i + 1, length - 1)];
|
||||
push((float) ((1 - frac) * a + frac * b));
|
||||
resamplePos += ratio;
|
||||
}
|
||||
resamplePos -= length;
|
||||
}
|
||||
|
||||
private void push(float sample) {
|
||||
pending[pendingCount++] = sample;
|
||||
if (pendingCount == RnnVad.FRAME_SIZE) {
|
||||
vad.process(pending, 0);
|
||||
pendingCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** Windowed-sinc low-pass with the given normalised cutoff, scaled to unit DC gain. */
|
||||
private static float[] lowPass(int taps, double cutoff) {
|
||||
float[] h = new float[taps];
|
||||
double centre = (taps - 1) / 2.0;
|
||||
double sum = 0;
|
||||
for (int i = 0; i < taps; i++) {
|
||||
double t = i - centre;
|
||||
double sinc = (t == 0) ? 2 * cutoff : Math.sin(2 * Math.PI * cutoff * t) / (Math.PI * t);
|
||||
double window = 0.54 - 0.46 * Math.cos(2 * Math.PI * i / (taps - 1)); // Hamming
|
||||
h[i] = (float) (sinc * window);
|
||||
sum += h[i];
|
||||
}
|
||||
for (int i = 0; i < taps; i++) {
|
||||
h[i] /= (float) sum;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package com.ts3client.audio.vad;
|
||||
|
||||
import static com.ts3client.audio.vad.VadConstants.BUF_SIZE_24K;
|
||||
import static com.ts3client.audio.vad.VadConstants.FEATURE_VECTOR_SIZE;
|
||||
import static com.ts3client.audio.vad.VadConstants.FRAME_10MS_24K;
|
||||
import static com.ts3client.audio.vad.VadConstants.FRAME_20MS_24K;
|
||||
import static com.ts3client.audio.vad.VadConstants.MAX_PITCH_24K;
|
||||
import static com.ts3client.audio.vad.VadConstants.NUM_LPC_COEFFICIENTS;
|
||||
|
||||
/**
|
||||
* WebRTC's {@code rnn_vad} speech-probability detector, ported to Java.
|
||||
*
|
||||
* <p>This is the detector the TeamSpeak 3 client uses for its "Automatic" and "Hybrid"
|
||||
* voice-activation modes: TS3 links the same network with byte-identical weights, so at a
|
||||
* given threshold this gates on the same frames theirs does.
|
||||
*
|
||||
* <p>Feed exactly {@link VadConstants#FRAME_10MS_24K} samples of 24 kHz mono audio in
|
||||
* [-1, 1] per call, in order. Combines {@code features_extraction.cc} and {@code rnn.cc}.
|
||||
*/
|
||||
public final class RnnVad {
|
||||
|
||||
/** Samples per call: 10 ms at 24 kHz. */
|
||||
public static final int FRAME_SIZE = FRAME_10MS_24K;
|
||||
|
||||
/** Sample rate this detector expects. */
|
||||
public static final int SAMPLE_RATE = VadConstants.SAMPLE_RATE_24K;
|
||||
|
||||
/**
|
||||
* WebRTC's audio processing carries float samples on the 16-bit integer scale, and the
|
||||
* network's thresholds (the silence cut-off, the pitch energy floor) are calibrated for
|
||||
* it. Callers pass [-1, 1], so scale on the way in.
|
||||
*/
|
||||
private static final float INT16_SCALE = 32768.0f;
|
||||
|
||||
/**
|
||||
* Rolling window of 24 kHz samples: one maximum pitch period of history followed by
|
||||
* the most recent 20 ms.
|
||||
*/
|
||||
private final float[] pitchBuffer = new float[BUF_SIZE_24K];
|
||||
private final float[] lpResidual = new float[BUF_SIZE_24K];
|
||||
private final float[] lpcCoeffs = new float[NUM_LPC_COEFFICIENTS];
|
||||
private final float[] featureVector = new float[FEATURE_VECTOR_SIZE];
|
||||
|
||||
private final LinearPrediction linearPrediction = new LinearPrediction();
|
||||
private final PitchEstimator pitchEstimator = new PitchEstimator();
|
||||
private final SpectralFeatures spectralFeatures = new SpectralFeatures();
|
||||
private final RnnNetwork network = new RnnNetwork();
|
||||
|
||||
private float probability;
|
||||
private int pitchPeriod48k;
|
||||
|
||||
/** Clears all history. Call on a capture discontinuity. */
|
||||
public void reset() {
|
||||
java.util.Arrays.fill(pitchBuffer, 0.0f);
|
||||
pitchEstimator.reset();
|
||||
pitchPeriod48k = 0;
|
||||
spectralFeatures.reset();
|
||||
network.reset();
|
||||
probability = 0.0f;
|
||||
}
|
||||
|
||||
/** Speech probability in [0, 1] from the most recent {@link #process} call. */
|
||||
public float probability() {
|
||||
return probability;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pitch period of the most recent frame, expressed at 48 kHz as the network's
|
||||
* feature is. Exposed for diagnostics and tests.
|
||||
*/
|
||||
public int pitchPeriod48kHz() {
|
||||
return pitchPeriod48k;
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyses one 10 ms frame and returns the speech probability in [0, 1].
|
||||
*
|
||||
* @param frame mono samples in [-1, 1]
|
||||
* @param offset index of the first sample
|
||||
*/
|
||||
public float process(float[] frame, int offset) {
|
||||
System.arraycopy(pitchBuffer, FRAME_10MS_24K, pitchBuffer, 0, BUF_SIZE_24K - FRAME_10MS_24K);
|
||||
int start = BUF_SIZE_24K - FRAME_10MS_24K;
|
||||
for (int i = 0; i < FRAME_10MS_24K; i++) {
|
||||
pitchBuffer[start + i] = frame[offset + i] * INT16_SCALE;
|
||||
}
|
||||
|
||||
// Search the pitch on the whitened signal so formants don't dominate the peak.
|
||||
linearPrediction.computeLpcCoefficients(pitchBuffer, lpcCoeffs);
|
||||
LinearPrediction.computeLpResidual(lpcCoeffs, pitchBuffer, lpResidual);
|
||||
pitchPeriod48k = pitchEstimator.estimate(lpResidual);
|
||||
// Normalisation based on the training-set statistics.
|
||||
featureVector[FEATURE_VECTOR_SIZE - 2] = 0.01f * (pitchPeriod48k - 300);
|
||||
|
||||
// Compare the most recent 20 ms against the 20 ms one pitch period earlier.
|
||||
int laggedOffset = MAX_PITCH_24K - pitchPeriod48k / 2;
|
||||
boolean silence = spectralFeatures.checkSilenceComputeFeatures(
|
||||
pitchBuffer, BUF_SIZE_24K - FRAME_20MS_24K, pitchBuffer, laggedOffset, featureVector);
|
||||
|
||||
probability = network.computeVadProbability(featureVector, silence);
|
||||
return probability;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package com.ts3client.audio.vad;
|
||||
|
||||
/**
|
||||
* Quantised weights of WebRTC's {@code rnn_vad} network, copied verbatim from
|
||||
* {@code third_party/rnnoise/src/rnn_vad_weights.cc} (BSD, Xiph/Jean-Marc Valin).
|
||||
*
|
||||
* <p>These are the same 4585 bytes the TeamSpeak 3 client links in, so our detector
|
||||
* produces the same speech probabilities theirs does.
|
||||
*
|
||||
* <p>Values are stored quantised; {@link #SCALE} converts them to float.
|
||||
*/
|
||||
final class RnnVadWeights {
|
||||
|
||||
/** Dequantisation factor applied to every table below. */
|
||||
static final float SCALE = 1.0f / 256.0f;
|
||||
|
||||
/** FC1 weights, laid out as [input][output] (42 x 24). */
|
||||
static final byte[] INPUT_DENSE_WEIGHTS = {
|
||||
-10, 0, -3, 1, -8, -6, 3, -13, 1, 0, -3, -7, -5, -3, 6, -1, -6, 0, -6, -4, -1, -2, 1, 1, -7, 2,
|
||||
21, 10, -5, -20, 24, 23, 37, 8, -2, 33, -6, 22, 13, -2, 50, 8, 13, 1, -15, 30, -10, 30, 0, 3, 5,
|
||||
27, 1, 4, -3, 41, 56, 35, -2, 49, -13, 11, 13, -2, -47, 5, -16, -60, -15, 77, -17, 26, -3, 14,
|
||||
-21, 19, -5, -19, -13, 0, 10, 14, 9, 31, -13, -41, -10, 4, 22, 18, -48, -6, -10, 62, -3, -18,
|
||||
-14, 12, 26, -28, 3, 14, 25, -13, -19, 6, 5, 36, -3, -65, -12, 0, 31, -7, -9, 101, -4, 26, 16,
|
||||
17, -12, -12, 14, -36, -3, 5, -15, 21, 2, 30, -3, 38, -4, 1, -6, 7, -7, 14, 38, -22, -30, -3,
|
||||
-7, 3, -39, -70, -126, 25, 34, 94, -67, -22, -33, 83, -47, -118, 4, 70, 33, 25, 62, -128, -76,
|
||||
-118, -113, 49, -12, -100, -18, -114, -33, 43, 32, 61, 40, -9, -106, 2, 36, -100, -40, -5, 20,
|
||||
-75, 61, -51, -9, 126, -27, -52, 5, -24, -21, -126, -114, -12, 15, 106, -2, 73, -125, 50, 13,
|
||||
-120, 35, 35, 4, -61, 29, -124, 6, -53, -69, -125, 64, -89, 36, -107, -103, -7, 27, 121, 69, 77,
|
||||
-35, 35, 95, -125, -49, 97, -45, -43, -23, 23, -28, -65, -118, 2, 8, -126, 27, -97, 92, 5, 55,
|
||||
82, 17, -57, -115, 37, 8, -106, -46, 41, -2, 21, -44, 8, -73, -58, -39, 34, 89, -95, 95, -117,
|
||||
120, -58, 31, 123, 1, -32, -109, -110, 60, -120, -43, -74, 5, 91, 26, 21, 114, 82, -83, -126,
|
||||
123, 22, -16, -67, 25, -83, 46, 48, -34, -121, -124, -63, -35, -9, 31, 82, 123, 6, -3, 117, 93,
|
||||
-2, -13, -36, 124, -112, -6, -102, -5, -33, -15, 44, -69, -127, -23, -40, -34, -85, 68, 83, -1,
|
||||
40, 8, 84, 118, -58, -55, -102, 123, -55, -14, -123, 44, -63, -14, 21, 35, 16, 24, -126, -13,
|
||||
-114, 35, 20, -36, 61, -9, 97, 34, 19, -32, -109, 76, -104, 99, -119, 45, -125, -51, -28, -8,
|
||||
-69, -8, 125, -45, -93, 113, 103, -41, -82, 52, 7, 126, 0, -40, 104, 55, -58, 17, -124, -93,
|
||||
-58, 8, -45, 1, 56, -123, 108, -47, -23, 115, 127, 17, -68, -13, 116, -82, -44, 45, 67, -120,
|
||||
-101, -15, -125, 120, -113, 17, -48, -73, 126, -64, -86, -118, -19, 112, -1, -66, -27, -62, 121,
|
||||
-86, -58, 50, 89, -38, -75, 95, -111, 12, -113, 2, -68, 2, -94, -121, 91, -5, 0, 79, 43, -7,
|
||||
-18, 79, 35, -38, 47, 1, -45, 83, -50, 102, 32, 55, -96, 15, -122, -69, 45, -27, 91, -62, -30,
|
||||
46, -95, 22, -72, -97, -1, 14, -122, 28, 127, 61, -126, 121, 9, 68, -120, 49, -60, 90, 3, 43,
|
||||
68, 54, 34, -10, 28, 21, -24, -54, 22, -113, -12, 82, -2, -17, -9, 127, 8, 116, -92, 0, -70,
|
||||
-33, 123, 66, 116, -74, -4, 74, -72, -22, -47, 1, -83, -60, -124, 1, 122, -57, -43, 49, 40,
|
||||
-126, -128, -8, -29, 28, -24, -123, -121, -70, -93, -37, -126, 11, -125, -37, 11, -31, -51,
|
||||
-124, 116, -128, 8, -25, 109, 75, -12, 7, 8, 10, 117, 124, -128, -128, 29, -26, 101, 21, -128,
|
||||
87, 8, -39, 23, -128, 127, -127, 74, -55, 74, 112, 127, 4, 55, 44, -92, 123, 34, -93, 47, -21,
|
||||
-92, 17, 49, -121, 92, 7, -126, -125, 124, -74, 3, -59, 18, -91, 3, -9, 9, 56, 116, 7, -29, 33,
|
||||
87, -21, -128, -13, 57, 74, 9, -29, -61, -97, -21, -95, -12, -114, 16, 82, 125, -7, 10, -24, 9,
|
||||
77, -128, -102, -25, 3, -126, 10, 13, -18, 51, 26, 127, -79, 35, 51, 12, -50, -24, 1, -7, 22,
|
||||
81, 65, 120, -30, -38, 85, 122, -4, -106, -11, 27, 53, 41, 8, -104, -66, -38, -124, 10, 12, 76,
|
||||
117, -109, 9, 11, 2, -18, 3, 113, -16, -79, -39, -123, -20, -128, 2, 13, -33, -58, 10, 84, -104,
|
||||
13, 64, 109, 1, 54, -12, 28, 24, 63, -126, 118, -82, 46, -12, -15, 14, -43, 60, 22, -32, -19,
|
||||
-46, 91, -107, 24, -94, 26, -47, 125, 6, 58, -15, -75, -26, -38, -35, 103, -16, -17, -13, 63,
|
||||
-2, 45, -45, -73, -23, 70, -87, 51, -17, 53, 76, 14, -18, -31, -14, 103, 8, 21, -28, -33, -20,
|
||||
-47, 6, 39, 40, -30, 7, -76, 55, 31, -20, -21, -59, 1, 25, -11, 17, 5, -13, -39, 0, -76, 50,
|
||||
-33, -29, -50, -16, -11, -12, -1, -46, 40, -10, 65, -19, 21, -41, -32, -83, -19, -4, 49, -60,
|
||||
118, -24, -46, 9, 102, -20, 8, -19, 25, 31, -3, -37, 0, 25, 7, 29, 2, -39, 127, -64, -20, 64,
|
||||
115, -30, 36, 100, 35, 122, 127, 127, -127, 127, -127, 19, 127, -89, -79, -32, 39, -127, 125,
|
||||
-80, 126, -127, 26, 8, 98, -8, -57, -90, -50, 126, 61, 127, -126, 40, -106, -68, 104, -125,
|
||||
-119, 11, 10, -127, 66, -56, -12, -126, -104, 27, 75, 38, -124, -126, -125, 84, -123, -45, -114,
|
||||
-128, 127, 103, -101, -124, 127, -11, -23, -123, 92, -123, 24, 126, 41, -2, -39, -27, -94, 40,
|
||||
-112, -48, 127, 58, 14, 38, -75, -64, 73, 117, 100, -119, -11, 6, 32, -126, -14, 35, 121, -10,
|
||||
54, -60, 89, -3, 69, -25, -20, 43, -86, -34, 24, 27, 7, -81, -99, -23, -16, -26, 13, 35, -97,
|
||||
80, -29, -13, -121, -12, -65, -94, 70, -89, -126, -95, 88, 33, 96, 29, -90, 69, 114, -78, 65,
|
||||
90, -47, -47, 89, 1, -12, 3, 8, 30, 5, 2, -30, -1, 6, -7, 10, -4, 46, -27, -40, 22, -6, -17, 45,
|
||||
24, -9, 23, -14, -63, -26, -12, -57, 27, 25, 55, -76, -47, 21, 34, 33, 26, 17, 14, 6, 9, 26, 25,
|
||||
-25, -25, -18
|
||||
};
|
||||
|
||||
/** FC1 bias (24). */
|
||||
static final byte[] INPUT_DENSE_BIAS = {
|
||||
38, -6, 127, 127, 127, -43, -127, 78, 127, 5, 127, 123, 127, 127, -128, -76, -126, 28, 127, 125,
|
||||
-30, 127, -89, -20
|
||||
};
|
||||
|
||||
/** GRU1 input weights, [input][gate][output] (24 x 3 x 24). */
|
||||
static final byte[] HIDDEN_GRU_WEIGHTS = {
|
||||
-124, 23, -123, -33, -95, -4, 8, -84, 4, 101, -119, 116, -4, 123, 103, -51, 29, -124, -114, -49,
|
||||
31, 9, 75, -128, 0, -49, 37, -50, 46, -21, -63, -104, 54, 82, 33, 21, 70, 127, -9, -79, -39,
|
||||
-23, -127, 107, 122, -96, -46, -18, -39, 13, -28, -48, 14, 56, -52, 49, -1, -121, 25, -18, -36,
|
||||
-52, -57, -30, 54, -124, -26, -47, 10, 39, 12, 2, 9, -127, -128, 102, 21, 11, -64, -71, 89,
|
||||
-113, -111, 54, 31, 94, 121, -40, 30, 40, -109, 73, -9, 108, -92, 2, -127, 116, 127, 127, -122,
|
||||
95, 127, -37, -127, 28, 89, 10, 24, -104, -62, -67, -14, 38, 14, -71, 22, -41, 20, -50, 39, 63,
|
||||
86, 127, -18, 79, 4, -51, 2, 33, 117, -113, -78, 56, -91, 37, 34, -45, -44, -22, 21, -16, 56,
|
||||
30, -84, -79, 38, -74, 127, 9, -25, 2, 82, 61, 25, -26, 26, 11, 117, -65, 12, -58, 42, -62, -93,
|
||||
11, 11, 124, -123, 80, -125, 11, -90, 42, 94, 4, -109, -1, 85, -52, 45, -26, -27, 77, -5, 30,
|
||||
90, 0, 95, -7, 53, 29, -82, 22, -9, 74, 2, -12, -73, 114, 97, -64, 122, -77, 43, 91, 86, 126,
|
||||
106, 72, 90, -43, 46, 96, -51, 21, 22, 68, 22, 41, 79, 75, -46, -105, 23, -116, 127, -123, 102,
|
||||
57, 85, 10, -29, 34, 125, 126, 124, 81, -15, 54, 96, -128, 39, -124, 103, 74, 126, 127, -50,
|
||||
-71, -122, -64, 93, -75, 71, 105, 122, 123, 126, 122, -127, 33, -63, -74, 124, -71, 33, 41, -56,
|
||||
19, 6, 65, 41, 90, -116, -3, -46, 75, -13, 98, -74, -42, 74, -95, -96, 81, 24, 32, -19, -123,
|
||||
74, 55, 109, 115, 0, 32, 33, 12, -20, 9, 127, 127, -61, 79, -48, -54, -49, 101, -9, 27, -106,
|
||||
74, 119, 77, 87, -126, -24, 127, 124, 31, 34, 127, 40, 3, -90, 127, 23, 57, -53, 127, -69, -88,
|
||||
-33, 127, 19, -46, -9, -125, 13, -126, -113, 127, -41, 46, 106, -62, 3, -10, 111, 49, -34, -24,
|
||||
-20, -112, 11, 101, -50, -34, 50, 65, -64, -106, 70, -48, 60, 9, -122, -45, 15, -112, -26, -4,
|
||||
1, 39, 23, 58, -45, -80, 127, 82, 58, 30, -94, -119, 51, -89, 95, -107, 30, 127, 125, 58, -52,
|
||||
-42, -38, -20, -122, 115, 39, -26, 5, 73, 13, -39, 43, -23, -20, -125, 23, 35, 53, -61, -66, 72,
|
||||
-20, 33, 8, 35, 4, 7, 18, 19, 16, -45, -50, -71, 31, -29, -41, -27, 10, 14, 27, 9, -23, 98, 6,
|
||||
-94, 92, 127, -114, 59, -26, -100, -62, -127, -17, -85, -60, 126, -42, -6, 33, -120, -26, -126,
|
||||
-127, -35, -114, -31, 25, -126, -100, -126, -64, -46, -31, 30, 25, -74, -111, -97, -81, -104,
|
||||
-114, -19, -9, -116, -69, 22, 30, 59, 8, -51, 16, -97, 18, -4, -89, 80, -50, 3, 36, -67, 56, 69,
|
||||
-26, 107, -10, 58, -28, -4, -57, -72, -111, 0, -75, -119, 14, -75, -49, -66, -49, 8, -121, 22,
|
||||
-54, 121, 30, 54, -26, -126, -123, 56, 5, 48, 21, -127, -11, 23, 25, -82, 6, -25, 119, 78, 4,
|
||||
-104, 27, 61, -48, 37, -13, -52, 50, -50, 44, -1, -22, -43, -59, -78, -67, -32, -26, 9, -3, 40,
|
||||
16, 19, 3, -9, 20, -6, -37, 28, 39, 17, -19, -10, 1, 6, -59, 74, 47, 3, -119, 0, -128, -107,
|
||||
-25, -22, -69, -23, -111, -42, -93, -120, 90, -85, -54, -118, 76, -79, 124, 101, -77, -75, -17,
|
||||
-71, -114, 68, 55, 79, -1, -123, -20, 127, -65, -123, -128, -87, 123, 9, -115, -14, 7, -4, 127,
|
||||
-79, -115, 125, -28, 89, -83, 49, 89, 119, -69, -5, 12, -49, 60, 57, -24, -99, -110, 76, -83,
|
||||
125, 73, 81, 11, 8, -45, 1, 83, 13, -70, -2, 97, 112, -97, 53, -9, -94, 124, 44, -49, -24, 52,
|
||||
76, -110, -70, -114, -12, 72, -4, -114, 43, -43, 81, 102, -84, -27, 62, -40, 52, 58, 124, -35,
|
||||
-51, -123, -43, 56, -75, -34, -35, -106, 93, -43, 14, -16, 46, 62, -97, 21, 30, -53, 21, -11,
|
||||
-33, -20, -95, 4, -126, 12, 45, 20, 108, 85, 11, 20, -40, 99, 4, -25, -18, -23, -12, -126, -55,
|
||||
-20, -44, -51, 91, -127, 127, -44, 7, 127, 78, 38, 125, -6, -94, -103, 73, 126, -126, 18, 59,
|
||||
-46, 106, 76, 116, -31, 75, -4, 92, 102, 32, -31, 73, 42, -21, -28, 57, 127, -8, -107, 115, 124,
|
||||
-94, -4, -128, 29, -57, 70, -82, 50, -13, -44, 38, 67, -93, 6, -39, -46, 56, 68, 27, 61, 26, 18,
|
||||
-72, 127, 22, 18, -31, 127, 61, -65, -38, 1, -67, -1, 8, -73, 46, -116, -94, 58, -49, 71, -40,
|
||||
-63, -82, -20, -60, 93, 76, 69, -106, 34, -31, 4, -25, 107, -18, 45, 4, -61, 126, 54, -126,
|
||||
-125, 41, 19, 44, 32, -98, 125, -24, 125, -96, -125, 15, 87, -4, -90, 18, -40, 28, -69, 67, 22,
|
||||
41, 39, 7, -48, -44, 12, 69, -13, 2, 44, -38, 111, -7, -126, -22, -9, 74, -128, -36, -7, -123,
|
||||
-15, -79, -91, -37, -127, -122, 104, 30, 7, 98, -37, 111, -116, -47, 127, -45, 118, -111, -123,
|
||||
-120, -77, -64, -125, 124, 77, 111, 77, 18, -113, 117, -9, 67, -77, 126, 49, -20, -124, 39, 41,
|
||||
-124, -34, 114, -87, -126, 98, -20, 59, -17, -24, 125, 107, 54, 35, 33, -44, 12, -29, 125, -71,
|
||||
-28, -63, -114, 28, -17, 121, -36, 127, 89, -122, -49, -18, -48, 17, 24, 19, -64, -128, 13, 86,
|
||||
45, 13, -49, 55, 84, 48, 80, -39, 99, -127, 70, -33, 30, 50, 126, -65, -117, -13, -20, -24, 127,
|
||||
115, -72, -104, 63, 126, -42, 57, 17, 46, 21, 119, 110, -100, -60, -112, 62, -33, 28, 26, -22,
|
||||
-60, -33, -54, 78, 25, 32, -114, 86, 44, 26, 43, 76, 121, 19, 97, -2, -3, -73, -68, 6, -116, 6,
|
||||
-43, -97, 46, -128, -120, -31, -119, -29, 16, 16, -126, -128, -126, -46, -9, -3, 92, -31, -76,
|
||||
-126, -3, -107, -12, -23, -69, 5, 51, 27, -42, 23, -70, -128, -29, 22, 29, -126, -55, 50, -71,
|
||||
-3, 127, 44, -27, -70, -63, -66, -70, 104, 86, 115, 29, -92, 41, -90, 44, -11, -28, 20, -11,
|
||||
-63, -16, 43, 31, 17, -73, -31, -1, -17, -11, -39, 56, 18, 124, 72, -14, 28, 69, -121, -125, 34,
|
||||
127, 63, 86, -80, -126, -125, -124, -47, 124, 77, 124, -19, 23, -7, -50, 96, -128, -93, 102,
|
||||
-53, -36, -87, 119, -125, 92, -126, 118, 102, 72, -2, 125, 10, 97, 124, -125, 125, 71, -20, -47,
|
||||
-116, -121, -4, -9, -32, 79, -124, -36, 33, -128, -74, 125, 23, 127, -29, -115, -32, 124, -89,
|
||||
32, -107, 43, -17, 24, 24, 18, 29, -13, -15, -36, 62, -91, 4, -41, 95, 28, -23, 6, 46, 84, 66,
|
||||
77, 68, -70, -1, -23, -6, 65, 70, -21, 9, 77, -12, 2, -118, 4, 9, -108, 84, 52, 2, 52, 13, -10,
|
||||
58, -110, 18, 66, -95, -23, 70, 31, -3, 56, 56, -3, -7, 1, -27, -48, -61, 41, -4, 10, -62, 32,
|
||||
-7, -24, 9, -48, -60, -4, 79, -20, -38, -76, 68, -49, -97, 0, -15, 5, -100, -49, -95, -99, -115,
|
||||
-9, -40, 10, 104, 13, 56, 127, -27, -109, -94, -118, -102, -44, -85, 52, 127, -4, 14, 62, 121,
|
||||
-122, -26, -79, -42, -34, 1, 25, -38, -79, -58, -31, -31, -90, -30, -123, 32, -56, 125, 66, 124,
|
||||
-1, 3, 91, -103, -7, 23, 78, -18, 9, 69, -69, 76, -38, -33, -2, -98, 18, 106, 84, 55, 87, -47,
|
||||
35, -124, 64, 41, -14, 46, 25, -2, 120, -21, 82, 19, -79, -37, -3, -8, -16, 21, 19, -5, -28,
|
||||
-112, 39, -6, -30, 53, -69, 53, 46, 127, 123, 78, 20, 28, -7, 73, 72, 17, -40, 41, 111, 57, 32,
|
||||
-95, 29, 28, -39, -65, 54, -20, -63, 29, -67, 3, 44, -57, -47, 11, 61, -22, -44, 61, 48, -100,
|
||||
20, 125, 96, -24, -16, 3, -69, -126, 74, -125, 9, 45, -67, -123, -59, -72, 118, 69, 45, 50, -57,
|
||||
67, 13, -66, -106, 47, 62, 22, -1, -22, -25, -40, -125, 3, 125, 32, 102, -56, -25, -75, -30,
|
||||
122, 60, -13, 36, -73, 7, -84, 124, 40, -118, 17, -87, -118, -8, 3, -27, 111, -40, 40, -51, 127,
|
||||
125, -45, -30, -54, 46, 80, -1, -30, 101, -17, 18, 26, 54, 7, -12, 1, -127, 123, -122, -27, -75,
|
||||
64, 10, 25, -15, -44, 127, -127, 5, -84, -81, -7, 19, -26, 126, 15, 116, -126, 14, -76, 44, 62,
|
||||
-110, -124, 125, -29, -87, -3, -69, 82, 90, 57, -123, 123, 100, -19, -51, -32, 69, 37, -57,
|
||||
-128, -124, -72, -13, 51, -7, -45, -73, 5, 99, -26, -117, -96, -109, 4, -31, -12, 0, 31, -42,
|
||||
-27, 12, -81, 118, 39, 83, 14, 41, -126, 107, -82, 94, -116, -122, -47, -109, -84, -128, -35,
|
||||
-56, 66, 8, -65, 19, 42, -46, -72, -109, 41, 43, -127, -113, 58, 127, 42, -75, -1, 65, 117, -55,
|
||||
-113, -123, 124, 43, -96, -115, -19, 68, 15, 94, 3, 75, 0, 34, 9, 42, 110, -48, 92, -76, 99,
|
||||
-17, 27, 32, 13, 125, 50, -17, 56, 4, 53, 34, -8, 99, 80, -126, -21, -65, -11, -46, 44, -81, -3,
|
||||
-121, 123, 66, -81, -84, 119, 127, 84, 105, 45, -66, -42, -23, 32, -25, 12, 111, 127, 88, 125,
|
||||
30, 24, -127, -9, -54, 127, -116, -119, 88, 70, 94, -120, 35, -93, 15, 22, -21, 25, -110, -123,
|
||||
-45, 8, -109, 125, -122, -86, -126, 8, -14, -120, -45, -45, 69, -125, -122, 6, 81, 86, 125, 95,
|
||||
54, 77, 54, -123, 126, -85, -117, 56, 11, 0, -61, -91, -12, -2, -113, -3, -15, -122, -63, -91,
|
||||
10, 84, -111, 125, 93, 21, 62, -78, -116, 13, -57, 28, -124, 126, 110, 12, 15, 95, 15, -19,
|
||||
-125, -97, 52, -7, 101, 9, 20, -125, -26, -56, 72, 77, 12, -126, 22, -29, 47, 62, 95, 112, 69,
|
||||
32, 97, -83, -8, -5, 67, -63, -123, 79, 59, 0, -6, -17, 4, -111, -52, 27, 65, 0
|
||||
};
|
||||
|
||||
/** GRU1 recurrent weights, same layout (24 x 3 x 24). */
|
||||
static final byte[] HIDDEN_GRU_RECURRENT_WEIGHTS = {
|
||||
65, 83, 35, 56, 24, -34, -28, -2, 125, 19, 42, -9, 124, -53, 24, -87, 11, 35, -81, -35, -125,
|
||||
-31, 123, -21, 33, -91, 113, -93, 45, -6, 53, 38, -92, 8, -27, 87, 4, 43, 43, 10, -128, -128,
|
||||
-46, 127, -38, -45, 25, -87, 19, 5, 52, -96, -23, -29, 121, -126, -24, -20, -2, 69, -50, 6, 71,
|
||||
-81, -125, 90, -94, 1, -38, 36, 89, 17, -60, 71, -48, 18, -15, 44, -18, 59, 11, 114, -51, 32,
|
||||
110, 1, 4, 109, -24, 127, 27, 60, 88, 24, 45, -59, 75, -36, 8, 57, -32, -25, 13, 126, -89, -61,
|
||||
-76, 127, 18, -62, -68, 23, -113, 5, 126, 43, -88, 26, -78, 18, 75, 21, 9, -74, 20, 41, 126,
|
||||
-118, -15, 9, 116, 126, -127, 34, -6, 126, -128, -53, -54, -55, -121, 70, 127, -12, -68, 82,
|
||||
-25, 104, -126, 126, -21, -26, 124, -75, -127, -120, 13, 61, -64, -108, -63, -65, -44, -35, -61,
|
||||
-39, 109, -74, 113, -3, 108, -30, 125, 120, 39, 125, -128, -95, -99, 111, 9, 25, 114, -75, -92,
|
||||
-54, -12, -32, -38, 10, 31, 10, 63, 51, 40, -99, 74, 4, 50, -128, -36, -35, -11, -28, -126, -7,
|
||||
66, -58, -126, -22, -83, -61, -127, 49, 126, -8, 7, 62, 36, -11, -32, -44, 63, 116, 41, 65,
|
||||
-127, 126, 63, -30, -96, 74, -92, 127, 38, -18, -128, 68, -5, 101, -4, 85, 58, 79, 0, -58, 8,
|
||||
119, -70, -1, -79, -68, 114, -28, -90, -6, -112, 2, 127, -8, 10, 55, -59, -126, 127, 125, 80,
|
||||
72, 35, -54, 95, -124, -124, 79, 23, -46, -61, -127, -100, 99, -77, 8, -87, 5, -2, 49, 85, 7,
|
||||
-71, 82, 53, -41, 22, -22, -93, -103, 6, 52, -56, 14, -8, -111, 85, 16, 54, 32, -118, -24, 61,
|
||||
-53, 96, -70, -5, -17, -67, -84, -7, -82, -107, -96, 21, -83, -58, 50, 12, -126, -1, -28, 34,
|
||||
-126, 115, 17, 91, 1, -127, 72, 11, 126, -81, 6, 96, -8, 77, 15, -6, 63, -27, 20, -123, -109,
|
||||
85, -79, -17, 126, -92, 2, -61, 20, 14, 17, 121, 123, 30, 57, 120, 127, 57, 42, 117, 98, 67, 39,
|
||||
-20, -70, 100, 7, 125, 122, 40, 16, -79, 125, 83, 41, -106, -57, 24, 55, 27, -66, -111, -44, -7,
|
||||
-43, -66, 121, 42, -128, -45, 35, 15, -127, 34, -35, -34, -40, -18, -6, 63, 111, 31, 116, 127,
|
||||
19, 24, -71, -39, 34, 11, 19, -40, 27, 12, 106, -10, 56, -82, -106, -2, -50, -52, 114, -126,
|
||||
-34, -43, -68, 10, 76, 57, -118, -128, 37, -104, 76, 125, 3, -76, 127, -29, 84, -94, -15, 55,
|
||||
125, 79, 127, -57, -125, 104, -68, 126, 126, -77, 51, 45, 33, -109, 115, -11, 1, 95, -121, -5,
|
||||
-9, -126, -114, 39, 68, -126, -107, -51, -42, 24, -8, 51, -27, -43, 66, -45, 62, -98, -109, 69,
|
||||
67, 0, -125, -128, 49, 31, 126, -122, 2, -55, -67, -126, -70, -128, -125, -77, 25, 16, -8, -102,
|
||||
11, -75, 82, 38, -5, 5, 19, 34, 47, -127, -93, 21, 24, -97, -18, 31, 39, 34, -20, 22, 123, 7,
|
||||
-77, -81, -46, -9, 1, 23, 39, -127, -43, -8, -50, 10, -21, 59, -9, -4, -13, -27, 44, 127, 52,
|
||||
-47, 70, -43, 52, 101, -49, 27, 45, 49, 33, -125, 55, 114, 20, -1, 76, -24, -96, 105, 24, 126,
|
||||
75, -21, -105, 13, -42, 40, 126, -30, -39, -95, 125, -63, 11, 6, 125, 125, -14, 5, 42, -61, -4,
|
||||
49, 88, 6, -107, -28, 19, -29, 47, 126, 6, -46, -89, -18, 91, -20, -6, 118, -21, -22, 39, 115,
|
||||
11, -42, 54, 73, -55, -77, 62, -27, -59, -99, -12, -127, -40, 56, -3, -124, -91, 71, -111, 6,
|
||||
-19, 82, -24, -35, 102, -42, 7, -126, -126, -125, 18, 98, -52, 127, 105, -52, 40, -83, 126,
|
||||
-122, 109, 5, 127, 48, 6, 5, -125, 100, -16, 29, 85, -89, 8, 4, 41, 62, -127, 62, 122, 85, 122,
|
||||
-107, 8, -125, 93, -127, 127, 102, 19, 19, -66, 41, -42, 114, 127, -48, -117, -29, -6, -73,
|
||||
-102, -3, -19, 0, 88, 42, 87, -117, -20, 2, 122, 28, 63, 71, 66, 120, 93, 124, -43, 49, 103, 31,
|
||||
90, -91, -22, -126, 26, -24, -21, 51, -126, 87, -103, -69, -10, -66, -23, 20, 97, 36, 25, -127,
|
||||
30, -20, -63, 30, 51, -116, 23, 40, -39, 36, -83, -77, -25, -50, 110, 14, 13, -109, 125, -65,
|
||||
-55, -87, 124, -126, -32, -72, -108, 127, 127, -125, -124, 61, 121, 102, -128, -127, 16, 100,
|
||||
127, -124, -68, 72, -93, -128, 43, -93, -19, -125, -97, -113, -33, 83, 127, -44, 127, -75, 127,
|
||||
16, 44, 50, -122, 23, 118, 46, 19, 26, -128, 10, 4, 99, -14, -82, -13, 30, 125, 57, 65, 60, -71,
|
||||
35, 98, 28, 7, 1, 43, 89, 70, 75, 121, -59, 82, -126, -53, -16, -116, -65, 52, -52, 0, 80, 35,
|
||||
45, -61, 46, 8, 107, 27, -26, -118, 90, 57, -10, 7, -15, 0, -39, -4, 12, 29, -1, 116, 84, 79,
|
||||
119, 125, -59, 28, -6, -25, -43, 2, 90, 79, 67, 103, -82, 2, -6, 125, 19, 73, 0, -105, 112, -17,
|
||||
104, 107, 124, 106, 19, 56, -44, 55, -112, 6, -39, -83, 126, -93, -98, 57, -120, -23, -38, 2,
|
||||
-31, -48, 106, 127, 127, 69, 16, 110, 71, 104, 62, -12, -22, 42, -37, -94, 34, -1, -32, -12,
|
||||
-124, -47, -13, 60, -75, -66, 58, -127, -2, 64, 76, -106, 73, -49, -31, 127, 126, 31, 16, 127,
|
||||
-110, 107, -16, -53, 20, 69, -14, -125, 59, -44, 15, 120, 125, 125, 43, 6, 19, -58, 127, 127,
|
||||
43, 16, 82, 97, -127, 127, -93, -41, 88, 0, 77, -15, 116, 16, -124, -31, -3, 95, -40, -126, -54,
|
||||
-126, -83, -8, -59, 6, 67, -29, 4, 124, -10, 112, -28, -8, 85, -21, 45, 84, 6, -8, 11, 72, 32,
|
||||
84, -62, 77, 2, -36, 75, 31, -50, 116, 126, 119, -88, -55, -14, -37, 126, 40, -108, -6, -6, 57,
|
||||
64, -28, -76, 30, -117, -93, 31, -92, -44, -64, 94, 58, 65, 114, 41, 47, 71, 42, -26, 99, -126,
|
||||
57, -5, 74, -19, -113, -1, 67, -21, 126, 1, -3, 33, 60, -82, 37, -48, 89, 114, -38, 127, -114,
|
||||
35, 58, -5, 21, -46, 121, -123, -43, 127, 115, 123, 122, -101, 126, 127, 81, 52, 89, -127, 102,
|
||||
42, 117, -9, -2, 125, 127, 110, 96, 120, 66, 70, 124, 55, 84, -38, -58, 119, -127, -16, -79,
|
||||
123, 18, -127, -50, -38, 120, -85, 1, 7, -56, 108, -77, -2, 21, 37, 1, 13, -105, -69, 28, -87,
|
||||
33, -104, -51, 126, 41, 3, -121, 28, 71, 58, 86, -8, 127, 94, -55, 125, 40, -19, 127, -33, -87,
|
||||
-23, 7, -111, -68, 9, 84, -119, 55, -82, 78, -37, -20, -9, -23, 53, -13, 15, -46, 116, 126,
|
||||
-127, 56, -126, 125, -7, -1, 45, 26, 125, 121, 29, 47, -86, 30, 10, 76, -125, -7, 23, 92, -12,
|
||||
-39, -18, 92, -97, -8, -85, -41, 49, -50, 123, -37, -126, -30, 14, 79, -49, -65, 9, -36, -38,
|
||||
-96, 85, -24, -13, 37, -25, -5, -64, -127, 55, -60, -18, -61, -63, 127, 56, 67, 15, 124, 72,
|
||||
120, 127, 40, -10, 114, 24, -23, 46, 78, -53, 125, 86, 124, 86, 0, 38, 93, 21, 127, 123, 75,
|
||||
-72, 13, 48, 33, 83, -51, 15, -32, -49, -33, 120, 64, 7, 9, 65, 60, 21, -21, -61, -53, -113, 84,
|
||||
-97, 101, 37, -114, -27, 41, 73, 126, -10, 59, 61, -15, 70, -13, 82, -4, 69, 56, 94, -91, -50,
|
||||
92, -74, -48, 53, -7, -107, 127, 28, 30, -26, -21, -61, 77, 82, 64, -91, -125, 122, -104, 127,
|
||||
123, 122, 123, 76, -126, 127, -6, -80, 7, 40, -66, -65, 54, -2, 23, 96, -64, 74, 2, -53, -12,
|
||||
-123, 39, 60, -20, 16, -17, -97, 23, -4, -53, -122, 32, -16, -54, -95, 43, 71, -1, -67, -33, 41,
|
||||
18, 72, 28, -83, 31, -100, -91, -27, 10, -128, -106, 2, 76, -13, 42, 34, 112, -19, 44, 40, -9,
|
||||
-11, 65, 92, -43, -125, 2, 47, -32, 25, 122, -29, 12, 101, -8, -126, -23, 43, 7, 125, -20, -124,
|
||||
82, -2, 13, -73, -106, 115, 31, 116, -23, -44, -71, 84, 3, 47, 91, 127, 127, -15, 95, 7, 93, 5,
|
||||
113, -50, 54, 11, 13, -127, 17, 72, 43, -23, 5, -70, 20, 15, -27, 99, 69, -109, -122, -94, 16,
|
||||
127, 0, 116, 104, 45, 108, -34, 87, 72, -14, 118, 46, 42, 109, -26, 95, 93, 127, 60, 127, -93,
|
||||
-54, -122, 34, -105, 56, 55, 103, 125, -71, -50, 95, -72, 127, 107, 21, 73, 126, 61, 127, 127,
|
||||
24, -62, 90, 73, 90, -46, -78, -124, 72, 123, -42, 50, -107, 17, -32, -62, -89, 124, 1, 80, -2,
|
||||
117, 119, -65, -127, -95, -121, -52, 103, 66, 75, -3, -62, -127, 127, -74, 124, 79, 49, 40, 105,
|
||||
-67, -71, -70, 43, 127, 119, -4, 66, 43, 23, 91, -126, 15, 63, -119, 112, 103, 15, -99, 31,
|
||||
-127, 69, 116, -46, -67, 2, -126, -29, 30, 30, -69, -98, -47, -87, -70, -127, 23, -73, 30, -7,
|
||||
94, -52, -65, 98, -45, 97, 53, 23, -9, -22, -52, -47, 6, -1, -85, -15, -61, -14, 68, 110, -10,
|
||||
-121, -25, -35, -15, -94, -123, 27, 75, 48, -66, -56, -44, 93, 109, 67, -36, 24, 70, -126, 8,
|
||||
-127, 126, 52, 11, -32, 120, -13, -26, -28, -125, 127, 106, -50, 124, 36, -126, -12, 0, -23, 76,
|
||||
-71, -126, -12, -17, -82, 12, 124, 57, 33, 4, 77, -46, 71, -34, 72, 125, -128, 124, -24, -128,
|
||||
75, -120, 69, -45, 55, 33, 127, -33, 4, -105, -41, -59, -91, 123, 44, -127, 127, -67, 52, 25,
|
||||
-125, -65, 100, -25, 123, 6, 11, -123, -92, -33, 126, -17, -4, 29, 33, 127, 96, 3, 87, -48, -18,
|
||||
-70, 123, 58, -127, -3, -52, -1, -36, -41, 127, 51, -52, -27, 46, -83, 57, 9, 126, 127, 94, 79,
|
||||
-37, -127, -40, 67, 52, 82, -66, 122, -13, -73, 127, -8, -80, 46, -48, 4, -54
|
||||
};
|
||||
|
||||
/** GRU1 bias, [gate][output] (3 x 24). */
|
||||
static final byte[] HIDDEN_GRU_BIAS = {
|
||||
124, 125, -57, -126, 53, 123, 127, -75, 68, 102, -2, 116, 124, 127, 124, 125, 126, 123, -16, 48,
|
||||
125, 126, 78, 85, 11, 126, -30, -30, -64, -3, -105, -29, -17, 69, 63, 2, -32, -10, -62, 113,
|
||||
-52, 112, -109, 112, 7, -40, 73, 53, 62, 6, -2, 0, 0, 100, -16, 26, -24, 56, 26, -10, -33, 41,
|
||||
70, 109, -29, 127, 34, -66, 49, 53, 27, 62
|
||||
};
|
||||
|
||||
/** FC2 weights (24 x 1). */
|
||||
static final byte[] OUTPUT_DENSE_WEIGHTS = {
|
||||
127, 127, 127, 127, 127, 20, 127, -126, -126, -54, 14, 125, -126, -126, 127, -125, -126, 127,
|
||||
-127, -127, -57, -30, 127, 80
|
||||
};
|
||||
|
||||
/** FC2 bias (1). */
|
||||
static final byte[] OUTPUT_DENSE_BIAS = {
|
||||
-50
|
||||
};
|
||||
|
||||
private RnnVadWeights() {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
package com.ts3client.audio.vad;
|
||||
|
||||
import static com.ts3client.audio.vad.VadConstants.CEPSTRAL_HISTORY;
|
||||
import static com.ts3client.audio.vad.VadConstants.FRAME_20MS_24K;
|
||||
import static com.ts3client.audio.vad.VadConstants.NUM_BANDS;
|
||||
import static com.ts3client.audio.vad.VadConstants.NUM_LOWER_BANDS;
|
||||
import static com.ts3client.audio.vad.VadConstants.OPUS_BANDS_24K;
|
||||
|
||||
/**
|
||||
* Band-wise spectral features for {@code rnn_vad}, ported from WebRTC's
|
||||
* {@code rnn_vad/spectral_features.cc} and {@code spectral_features_internal.cc}.
|
||||
*
|
||||
* <p>A 20 ms Vorbis-windowed FFT is folded into 22 Opus-scale bands, log-compressed
|
||||
* and DCT'd into a cepstrum. The network is fed the higher cepstral coefficients, the
|
||||
* running average and first two derivatives of the lower ones, the cepstrum of the
|
||||
* band-wise correlation between the current frame and the frame one pitch period earlier,
|
||||
* and a measure of how much the cepstrum has been varying.
|
||||
*/
|
||||
final class SpectralFeatures {
|
||||
|
||||
/** Total band energy below which the frame is treated as silence. */
|
||||
private static final float SILENCE_THRESHOLD = 0.04f;
|
||||
|
||||
/** FFT bins per Opus band at 24 kHz for a 20 ms frame. */
|
||||
private static final int[] OPUS_BAND_SIZES =
|
||||
{4, 4, 4, 4, 4, 4, 4, 4, 8, 8, 8, 8, 16, 16, 16, 24, 24, 32, 48};
|
||||
|
||||
/** sqrt(2 / NUM_BANDS), applied after the DCT. */
|
||||
private static final float DCT_SCALING_FACTOR = 0.301511345f;
|
||||
|
||||
private final RealFft fft = new RealFft(FRAME_20MS_24K);
|
||||
private final float[] halfWindow = new float[FRAME_20MS_24K / 2];
|
||||
private final float[] bandWeights = new float[FRAME_20MS_24K / 2];
|
||||
private final float[] dctTable = new float[NUM_BANDS * NUM_BANDS];
|
||||
|
||||
private final float[] windowed = new float[FRAME_20MS_24K];
|
||||
private final float[] referenceFft = new float[FRAME_20MS_24K];
|
||||
private final float[] laggedFft = new float[FRAME_20MS_24K];
|
||||
private final float[] referenceBandsEnergy = new float[OPUS_BANDS_24K];
|
||||
private final float[] laggedBandsEnergy = new float[OPUS_BANDS_24K];
|
||||
private final float[] bandsCrossCorr = new float[OPUS_BANDS_24K];
|
||||
private final float[] logBandsEnergy = new float[NUM_BANDS];
|
||||
private final float[] cepstrum = new float[NUM_BANDS];
|
||||
private final float[] crossCorrCepstrum = new float[NUM_LOWER_BANDS];
|
||||
|
||||
/** Ring buffer of the last {@link VadConstants#CEPSTRAL_HISTORY} cepstra. */
|
||||
private final float[][] cepstralHistory = new float[CEPSTRAL_HISTORY][NUM_BANDS];
|
||||
private int cepstralTail;
|
||||
|
||||
/**
|
||||
* Upper-right triangle of pairwise cepstral distances, stored as a square matrix so a
|
||||
* new frame can be shifted in with a single array copy.
|
||||
*/
|
||||
private final float[] cepstralDiffs = new float[(CEPSTRAL_HISTORY - 1) * (CEPSTRAL_HISTORY - 1)];
|
||||
private final float[] newDistances = new float[CEPSTRAL_HISTORY - 1];
|
||||
|
||||
SpectralFeatures() {
|
||||
int halfSize = FRAME_20MS_24K / 2;
|
||||
float scaling = 1.0f / FRAME_20MS_24K;
|
||||
for (int i = 0; i < halfSize; i++) {
|
||||
double inner = Math.sin(0.5 * Math.PI * (i + 0.5) / halfSize);
|
||||
halfWindow[i] = (float) (scaling * Math.sin(0.5 * Math.PI * inner * inner));
|
||||
}
|
||||
// Each band overlaps its neighbour with a triangular response; the weight is the
|
||||
// fractional position of the bin within its band.
|
||||
int bin = 0;
|
||||
for (int bandSize : OPUS_BAND_SIZES) {
|
||||
for (int j = 0; j < bandSize; j++) {
|
||||
bandWeights[bin++] = (float) j / bandSize;
|
||||
}
|
||||
}
|
||||
double k = Math.sqrt(0.5);
|
||||
for (int i = 0; i < NUM_BANDS; i++) {
|
||||
for (int j = 0; j < NUM_BANDS; j++) {
|
||||
dctTable[i * NUM_BANDS + j] = (float) Math.cos((i + 0.5) * j * Math.PI / NUM_BANDS);
|
||||
}
|
||||
dctTable[i * NUM_BANDS] *= (float) k;
|
||||
}
|
||||
}
|
||||
|
||||
void reset() {
|
||||
for (float[] row : cepstralHistory) {
|
||||
java.util.Arrays.fill(row, 0.0f);
|
||||
}
|
||||
cepstralTail = 0;
|
||||
java.util.Arrays.fill(cepstralDiffs, 0.0f);
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the spectral part of the feature vector and returns {@code true} if the
|
||||
* reference frame is silence, in which case the vector is only partially written.
|
||||
*
|
||||
* @param reference buffer holding the most recent 20 ms at 24 kHz
|
||||
* @param referenceOffset offset of that frame within {@code reference}
|
||||
* @param lagged buffer holding the 20 ms one pitch period earlier
|
||||
* @param laggedOffset offset of that frame within {@code lagged}
|
||||
*/
|
||||
boolean checkSilenceComputeFeatures(float[] reference, int referenceOffset,
|
||||
float[] lagged, int laggedOffset,
|
||||
float[] featureVector) {
|
||||
forwardFft(reference, referenceOffset, referenceFft);
|
||||
crossCorrelation(referenceFft, referenceFft, referenceBandsEnergy);
|
||||
|
||||
float totalEnergy = 0;
|
||||
for (float e : referenceBandsEnergy) {
|
||||
totalEnergy += e;
|
||||
}
|
||||
if (totalEnergy < SILENCE_THRESHOLD) {
|
||||
return true;
|
||||
}
|
||||
|
||||
forwardFft(lagged, laggedOffset, laggedFft);
|
||||
crossCorrelation(laggedFft, laggedFft, laggedBandsEnergy);
|
||||
|
||||
smoothedLogMagnitudeSpectrum(referenceBandsEnergy, logBandsEnergy);
|
||||
dct(logBandsEnergy, NUM_BANDS, cepstrum, NUM_BANDS);
|
||||
// Ad-hoc correction terms for the first two cepstral coefficients.
|
||||
cepstrum[0] -= 12.0f;
|
||||
cepstrum[1] -= 4.0f;
|
||||
|
||||
pushCepstrum(cepstrum);
|
||||
|
||||
// Higher bands go in as-is; the lower ones are summarised over time instead.
|
||||
System.arraycopy(cepstrum, NUM_LOWER_BANDS, featureVector, NUM_LOWER_BANDS,
|
||||
NUM_BANDS - NUM_LOWER_BANDS);
|
||||
computeAverageAndDerivatives(featureVector);
|
||||
computeNormalizedCepstralCorrelation(featureVector);
|
||||
featureVector[VadConstants.FEATURE_VECTOR_SIZE - 1] = computeVariability();
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Windows a 20 ms frame and transforms it, zeroing the unused Nyquist coefficient. */
|
||||
private void forwardFft(float[] frame, int offset, float[] out) {
|
||||
int halfSize = FRAME_20MS_24K / 2;
|
||||
for (int i = 0, j = FRAME_20MS_24K - 1; i < halfSize; i++, j--) {
|
||||
windowed[i] = frame[offset + i] * halfWindow[i];
|
||||
windowed[j] = frame[offset + j] * halfWindow[i];
|
||||
}
|
||||
fft.forward(windowed, out);
|
||||
out[1] = 0.0f;
|
||||
}
|
||||
|
||||
/**
|
||||
* Folds the interleaved spectra into Opus bands. Each bin contributes to the band it
|
||||
* sits in and to the next one, split by the triangular weight.
|
||||
*/
|
||||
private void crossCorrelation(float[] x, float[] y, float[] out) {
|
||||
int k = 0;
|
||||
out[0] = 0.0f;
|
||||
for (int i = 0; i < OPUS_BANDS_24K - 1; i++) {
|
||||
out[i + 1] = 0.0f;
|
||||
for (int j = 0; j < OPUS_BAND_SIZES[i]; j++) {
|
||||
float v = x[2 * k] * y[2 * k] + x[2 * k + 1] * y[2 * k + 1];
|
||||
float tmp = bandWeights[k] * v;
|
||||
out[i] += v - tmp;
|
||||
out[i + 1] += tmp;
|
||||
k++;
|
||||
}
|
||||
}
|
||||
out[0] *= 2.0f; // The first band only receives half a triangle.
|
||||
}
|
||||
|
||||
/** Log magnitude with smoothing over frequency and a decaying floor. */
|
||||
private static void smoothedLogMagnitudeSpectrum(float[] bandsEnergy, float[] out) {
|
||||
final float logOneByHundred = -2.0f;
|
||||
float logMax = logOneByHundred;
|
||||
float follow = logOneByHundred;
|
||||
for (int i = 0; i < out.length; i++) {
|
||||
float x = (i < bandsEnergy.length)
|
||||
? (float) Math.log10(0.01f + bandsEnergy[i])
|
||||
: logOneByHundred;
|
||||
x = Math.max(logMax - 7.0f, Math.max(follow - 1.5f, x));
|
||||
logMax = Math.max(logMax, x);
|
||||
follow = Math.max(follow - 1.5f, x);
|
||||
out[i] = x;
|
||||
}
|
||||
}
|
||||
|
||||
/** DCT-II of the first {@code inSize} values of {@code in}, truncated to {@code outSize}. */
|
||||
private void dct(float[] in, int inSize, float[] out, int outSize) {
|
||||
for (int i = 0; i < outSize; i++) {
|
||||
float sum = 0;
|
||||
for (int j = 0; j < inSize; j++) {
|
||||
sum += in[j] * dctTable[j * NUM_BANDS + i];
|
||||
}
|
||||
out[i] = sum * DCT_SCALING_FACTOR;
|
||||
}
|
||||
}
|
||||
|
||||
private void pushCepstrum(float[] newCepstrum) {
|
||||
// Distance from the new cepstrum to each older one, nearest first.
|
||||
for (int i = 0; i < CEPSTRAL_HISTORY - 1; i++) {
|
||||
float[] old = cepstralHistoryAt(i);
|
||||
float sum = 0;
|
||||
for (int k = 0; k < NUM_BANDS; k++) {
|
||||
float c = newCepstrum[k] - old[k];
|
||||
sum += c * c;
|
||||
}
|
||||
newDistances[i] = sum;
|
||||
}
|
||||
System.arraycopy(newCepstrum, 0, cepstralHistory[cepstralTail], 0, NUM_BANDS);
|
||||
cepstralTail = (cepstralTail + 1) % CEPSTRAL_HISTORY;
|
||||
pushCepstralDistances();
|
||||
}
|
||||
|
||||
/** Returns the cepstrum pushed {@code delay} frames ago (0 = most recent). */
|
||||
private float[] cepstralHistoryAt(int delay) {
|
||||
int offset = cepstralTail - 1 - delay;
|
||||
if (offset < 0) {
|
||||
offset += CEPSTRAL_HISTORY;
|
||||
}
|
||||
return cepstralHistory[offset];
|
||||
}
|
||||
|
||||
/** Shifts the distance matrix up-left by one and writes the new column. */
|
||||
private void pushCepstralDistances() {
|
||||
final int s = CEPSTRAL_HISTORY;
|
||||
System.arraycopy(cepstralDiffs, s, cepstralDiffs, 0, cepstralDiffs.length - s);
|
||||
for (int i = 0; i < newDistances.length; i++) {
|
||||
cepstralDiffs[(s - 1 - i) * (s - 1) - 1] = newDistances[i];
|
||||
}
|
||||
}
|
||||
|
||||
private float cepstralDistance(int delay1, int delay2) {
|
||||
int row = CEPSTRAL_HISTORY - 1 - delay1;
|
||||
int col = CEPSTRAL_HISTORY - 1 - delay2;
|
||||
if (row > col) { // Only the upper-right triangle is stored.
|
||||
int tmp = row;
|
||||
row = col;
|
||||
col = tmp;
|
||||
}
|
||||
return cepstralDiffs[row * (CEPSTRAL_HISTORY - 1) + (col - 1)];
|
||||
}
|
||||
|
||||
private void computeAverageAndDerivatives(float[] featureVector) {
|
||||
float[] curr = cepstralHistoryAt(0);
|
||||
float[] prev1 = cepstralHistoryAt(1);
|
||||
float[] prev2 = cepstralHistoryAt(2);
|
||||
for (int i = 0; i < NUM_LOWER_BANDS; i++) {
|
||||
featureVector[i] = curr[i] + prev1[i] + prev2[i]; // kernel [1, 1, 1]
|
||||
featureVector[NUM_BANDS + i] = curr[i] - prev2[i]; // kernel [1, 0, -1]
|
||||
featureVector[NUM_BANDS + NUM_LOWER_BANDS + i] =
|
||||
curr[i] - 2 * prev1[i] + prev2[i]; // kernel [1, -2, 1]
|
||||
}
|
||||
}
|
||||
|
||||
/** Band correlation between the current frame and the one a pitch period earlier. */
|
||||
private void computeNormalizedCepstralCorrelation(float[] featureVector) {
|
||||
crossCorrelation(referenceFft, laggedFft, bandsCrossCorr);
|
||||
for (int i = 0; i < bandsCrossCorr.length; i++) {
|
||||
bandsCrossCorr[i] /= (float) Math.sqrt(
|
||||
0.001f + referenceBandsEnergy[i] * laggedBandsEnergy[i]);
|
||||
}
|
||||
dct(bandsCrossCorr, OPUS_BANDS_24K, crossCorrCepstrum, NUM_LOWER_BANDS);
|
||||
// Ad-hoc correction terms, as for the frame cepstrum.
|
||||
crossCorrCepstrum[0] -= 1.3f;
|
||||
crossCorrCepstrum[1] -= 0.9f;
|
||||
System.arraycopy(crossCorrCepstrum, 0, featureVector,
|
||||
NUM_BANDS + 2 * NUM_LOWER_BANDS, NUM_LOWER_BANDS);
|
||||
}
|
||||
|
||||
/**
|
||||
* How far the current cepstrum sits from its nearest neighbour in the recent history,
|
||||
* averaged over the history. Steady tones score low, speech scores high.
|
||||
*/
|
||||
private float computeVariability() {
|
||||
float variability = 0;
|
||||
for (int delay1 = 0; delay1 < CEPSTRAL_HISTORY; delay1++) {
|
||||
float minDist = Float.MAX_VALUE;
|
||||
for (int delay2 = 0; delay2 < CEPSTRAL_HISTORY; delay2++) {
|
||||
if (delay1 == delay2) {
|
||||
continue; // The distance would be 0.
|
||||
}
|
||||
minDist = Math.min(minDist, cepstralDistance(delay1, delay2));
|
||||
}
|
||||
variability += minDist;
|
||||
}
|
||||
// Normalised against training-set statistics.
|
||||
return variability / CEPSTRAL_HISTORY - 2.1f;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.ts3client.audio.vad;
|
||||
|
||||
/**
|
||||
* Sizes and pitch-search bounds of WebRTC's {@code rnn_vad}, mirroring
|
||||
* {@code modules/audio_processing/agc2/rnn_vad/common.h}.
|
||||
*
|
||||
* <p>The detector analyses 10 ms frames at 24 kHz. Spectral analysis uses the
|
||||
* most recent 20 ms; pitch search additionally needs one maximum pitch period of
|
||||
* history, hence the 864-sample buffer.
|
||||
*/
|
||||
final class VadConstants {
|
||||
|
||||
static final int SAMPLE_RATE_24K = 24_000;
|
||||
static final int FRAME_10MS_24K = SAMPLE_RATE_24K / 100; // 240
|
||||
static final int FRAME_20MS_24K = FRAME_10MS_24K * 2; // 480
|
||||
|
||||
/** Pitch range at 24 kHz: 0.00125 s to 0.016 s. */
|
||||
static final int MIN_PITCH_24K = SAMPLE_RATE_24K / 800; // 30
|
||||
static final int MAX_PITCH_24K = 384; // 24000 / 62.5
|
||||
static final int BUF_SIZE_24K = MAX_PITCH_24K + FRAME_20MS_24K; // 864
|
||||
|
||||
/**
|
||||
* The coarse search skips very short periods; a refinement step recovers them, so the
|
||||
* initial pass starts three minimum periods in.
|
||||
*/
|
||||
static final int INITIAL_MIN_PITCH_24K = 3 * MIN_PITCH_24K; // 90
|
||||
static final int INITIAL_NUM_LAGS_24K = MAX_PITCH_24K - INITIAL_MIN_PITCH_24K; // 294
|
||||
static final int REFINE_NUM_LAGS_24K = MAX_PITCH_24K + 1; // 385
|
||||
|
||||
// 12 kHz analysis, used for the coarse pitch search.
|
||||
static final int FRAME_20MS_12K = 240;
|
||||
static final int BUF_SIZE_12K = BUF_SIZE_24K / 2; // 432
|
||||
static final int INITIAL_MIN_PITCH_12K = INITIAL_MIN_PITCH_24K / 2; // 45
|
||||
static final int MAX_PITCH_12K = MAX_PITCH_24K / 2; // 192
|
||||
static final int NUM_LAGS_12K = MAX_PITCH_12K - INITIAL_MIN_PITCH_12K; // 147
|
||||
|
||||
// The pitch period fed to the network is expressed at 48 kHz.
|
||||
static final int MIN_PITCH_48K = MIN_PITCH_24K * 2; // 60
|
||||
static final int MAX_PITCH_48K = MAX_PITCH_24K * 2; // 768
|
||||
|
||||
// Spectral features.
|
||||
static final int NUM_BANDS = 22;
|
||||
static final int NUM_LOWER_BANDS = 6;
|
||||
static final int CEPSTRAL_HISTORY = 8;
|
||||
/** Opus bands that fall below the 24 kHz Nyquist frequency. */
|
||||
static final int OPUS_BANDS_24K = 20;
|
||||
|
||||
static final int NUM_LPC_COEFFICIENTS = 5;
|
||||
|
||||
static final int FEATURE_VECTOR_SIZE = 42;
|
||||
|
||||
private VadConstants() {
|
||||
}
|
||||
}
|
||||
@@ -69,9 +69,15 @@ public final class Settings {
|
||||
public InputMode inputMode = InputMode.VOICE_ACTIVATION;
|
||||
/** Voice-activation strategy used when {@link #inputMode} is VOICE_ACTIVATION. */
|
||||
public VadMode vadMode = VadMode.HYBRID;
|
||||
/** VAD threshold in dBFS (RMS). Typical range -60 (sensitive) .. -10 (loud). */
|
||||
public double vadThresholdDb = -45.0;
|
||||
/** Speech-probability threshold (0..1) for Automatic/Hybrid modes. */
|
||||
/**
|
||||
* Volume-gate threshold on {@link com.ts3client.audio.InputLevel}'s scale, the same one
|
||||
* the TS3 client's slider uses: -50 (most sensitive) .. +50, defaulting to -40.
|
||||
*/
|
||||
public double vadThresholdDb = -40.0;
|
||||
/**
|
||||
* Speech-probability threshold (0..1) for Automatic/Hybrid modes. TS3 calls this
|
||||
* {@code vad_likelihood} and also defaults it to 0.5.
|
||||
*/
|
||||
public double speechThreshold = 0.5;
|
||||
/** Keep voice activation running while in push-to-talk mode. */
|
||||
public boolean vadOverPtt = false;
|
||||
@@ -168,7 +174,8 @@ public final class Settings {
|
||||
outputDevice = props.getProperty("outputDevice", outputDevice);
|
||||
inputMode = parseMode(props.getProperty("inputMode"), inputMode);
|
||||
vadMode = parseVadMode(props.getProperty("vadMode"), vadMode);
|
||||
vadThresholdDb = parseD(props.getProperty("vadThresholdDb"), vadThresholdDb);
|
||||
vadThresholdDb = parseD(props.getProperty("vadLevelDb"),
|
||||
migrateThreshold(props.getProperty("vadThresholdDb"), vadThresholdDb));
|
||||
speechThreshold = parseD(props.getProperty("speechThreshold"), speechThreshold);
|
||||
vadOverPtt = parseB(props.getProperty("vadOverPtt"), vadOverPtt);
|
||||
bitrate = parseI(props.getProperty("bitrate"), bitrate);
|
||||
@@ -205,7 +212,8 @@ public final class Settings {
|
||||
props.setProperty("outputDevice", outputDevice);
|
||||
props.setProperty("inputMode", inputMode.name());
|
||||
props.setProperty("vadMode", vadMode.name());
|
||||
props.setProperty("vadThresholdDb", Double.toString(vadThresholdDb));
|
||||
props.setProperty("vadLevelDb", Double.toString(vadThresholdDb));
|
||||
props.remove("vadThresholdDb");
|
||||
props.setProperty("speechThreshold", Double.toString(speechThreshold));
|
||||
props.setProperty("vadOverPtt", Boolean.toString(vadOverPtt));
|
||||
props.setProperty("bitrate", Integer.toString(bitrate));
|
||||
@@ -246,6 +254,20 @@ public final class Settings {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a threshold saved under the old {@code vadThresholdDb} key, which was in
|
||||
* dBFS ({@code 20*log10(rms)}), to {@link com.ts3client.audio.InputLevel}'s scale, so
|
||||
* upgrading doesn't silently change how sensitive the gate is.
|
||||
*/
|
||||
private static double migrateThreshold(String legacy, double def) {
|
||||
double dbfs = parseD(legacy, Double.NaN);
|
||||
if (Double.isNaN(dbfs)) {
|
||||
return def;
|
||||
}
|
||||
double converted = com.ts3client.audio.InputLevel.powerToDb(Math.pow(10.0, dbfs / 10.0));
|
||||
return Math.min(com.ts3client.audio.InputLevel.MAX_DB, converted);
|
||||
}
|
||||
|
||||
private static double parseD(String v, double def) {
|
||||
if (v == null) return def;
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package com.ts3client.audio.vad;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Random;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class RealFftTest {
|
||||
|
||||
/**
|
||||
* The rnn_vad transform length is 480 = 2^5 * 3 * 5, so the mixed-radix path has to be
|
||||
* right for factors other than powers of two.
|
||||
*/
|
||||
@Test
|
||||
void matchesDirectDftForMixedRadixLengths() {
|
||||
for (int n : new int[]{480, 240, 512, 64, 60, 210, 100}) {
|
||||
RealFft fft = new RealFft(n);
|
||||
Random random = new Random(n);
|
||||
float[] in = new float[n];
|
||||
float[] out = new float[n];
|
||||
for (int i = 0; i < n; i++) {
|
||||
in[i] = (float) random.nextGaussian();
|
||||
}
|
||||
fft.forward(in, out);
|
||||
|
||||
for (int k = 0; k <= n / 2; k++) {
|
||||
double re = 0;
|
||||
double im = 0;
|
||||
for (int t = 0; t < n; t++) {
|
||||
double angle = -2 * Math.PI * k * t / n;
|
||||
re += in[t] * Math.cos(angle);
|
||||
im += in[t] * Math.sin(angle);
|
||||
}
|
||||
double gotRe;
|
||||
double gotIm;
|
||||
if (k == 0) {
|
||||
gotRe = out[0];
|
||||
gotIm = 0;
|
||||
} else if (k == n / 2) {
|
||||
gotRe = out[1]; // Packed layout keeps Nyquist in slot 1.
|
||||
gotIm = 0;
|
||||
} else {
|
||||
gotRe = out[2 * k];
|
||||
gotIm = out[2 * k + 1];
|
||||
}
|
||||
assertEquals(re, gotRe, 1e-2, "n=" + n + " bin=" + k + " real");
|
||||
assertEquals(im, gotIm, 1e-2, "n=" + n + " bin=" + k + " imag");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsOddLengths() {
|
||||
org.junit.jupiter.api.Assertions.assertThrows(
|
||||
IllegalArgumentException.class, () -> new RealFft(481));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package com.ts3client.audio.vad;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Random;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Behavioural checks on the {@code rnn_vad} port. The network weights are the ones the TS3
|
||||
* client links, so these also pin the detector's agreement with it.
|
||||
*/
|
||||
class RnnVadTest {
|
||||
|
||||
private static final int FRAMES = 60;
|
||||
|
||||
@Test
|
||||
void silenceScoresZero() {
|
||||
float[] x = new float[RnnVad.FRAME_SIZE * FRAMES];
|
||||
double[] probabilities = run(x);
|
||||
for (double p : probabilities) {
|
||||
assertEquals(0.0, p, 0.0, "silence must not score above zero");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void voicedSpeechLikeSignalScoresHigh() {
|
||||
float[] x = synth(FRAMES, t -> {
|
||||
double sum = 0;
|
||||
for (int h = 1; h <= 30 && 120 * h < 8000; h++) {
|
||||
double f = 120 * h;
|
||||
double gain = formant(f, 700, 90) + formant(f, 1220, 110) + formant(f, 2600, 160);
|
||||
sum += gain * Math.sin(2 * Math.PI * f * t + h);
|
||||
}
|
||||
// A slow tremolo keeps it from looking like a stationary tone.
|
||||
return 0.25 * (0.6 + 0.4 * Math.sin(2 * Math.PI * 5 * t)) * sum / 6.0;
|
||||
});
|
||||
// Skip the first frames: the pitch buffer and GRU need history.
|
||||
assertTrue(meanFrom(run(x), 20) > 0.8, "a voiced signal should score high");
|
||||
}
|
||||
|
||||
@Test
|
||||
void quietPinkNoiseScoresLow() {
|
||||
Random random = new Random(3);
|
||||
float[] x = new float[RnnVad.FRAME_SIZE * FRAMES];
|
||||
double b0 = 0;
|
||||
double b1 = 0;
|
||||
double b2 = 0;
|
||||
for (int i = 0; i < x.length; i++) {
|
||||
double w = random.nextGaussian();
|
||||
b0 = 0.99765 * b0 + w * 0.0990460;
|
||||
b1 = 0.96300 * b1 + w * 0.2965164;
|
||||
b2 = 0.57000 * b2 + w * 1.0526913;
|
||||
// About -45 dBFS: representative of room tone rather than loud hiss.
|
||||
x[i] = (float) ((b0 + b1 + b2 + w * 0.1848) * 0.0011);
|
||||
}
|
||||
assertTrue(meanFrom(run(x), 20) < 0.2, "quiet room tone should not read as speech");
|
||||
}
|
||||
|
||||
@Test
|
||||
void estimatesPitchPeriodExactly() {
|
||||
// The pitch feature is fed to the network as 0.01 * (period48k - 300).
|
||||
for (double f0 : new double[]{80, 100, 120, 150, 200, 250, 300, 400}) {
|
||||
RnnVad vad = new RnnVad();
|
||||
float[] x = synth(FRAMES, t -> {
|
||||
double sum = 0;
|
||||
for (int h = 1; h <= 40 && f0 * h < 11000; h++) {
|
||||
sum += Math.sin(2 * Math.PI * f0 * h * t) / h;
|
||||
}
|
||||
return 0.2 * sum;
|
||||
});
|
||||
for (int f = 0; f < FRAMES; f++) {
|
||||
vad.process(x, f * RnnVad.FRAME_SIZE);
|
||||
}
|
||||
assertEquals(48000 / f0, vad.pitchPeriod48kHz(), 1e-9,
|
||||
"pitch period for f0=" + f0);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void resetClearsState() {
|
||||
float[] x = synth(FRAMES, t -> 0.3 * Math.sin(2 * Math.PI * 200 * t));
|
||||
RnnVad vad = new RnnVad();
|
||||
for (int f = 0; f < FRAMES; f++) {
|
||||
vad.process(x, f * RnnVad.FRAME_SIZE);
|
||||
}
|
||||
assertTrue(vad.probability() > 0.0);
|
||||
vad.reset();
|
||||
assertEquals(0.0, vad.probability(), 0.0);
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
private interface Signal {
|
||||
double at(double t);
|
||||
}
|
||||
|
||||
private static double formant(double f, double centre, double bandwidth) {
|
||||
double d = (f - centre) / bandwidth;
|
||||
return 1.0 / (1.0 + d * d);
|
||||
}
|
||||
|
||||
private static float[] synth(int frames, Signal signal) {
|
||||
float[] x = new float[RnnVad.FRAME_SIZE * frames];
|
||||
for (int i = 0; i < x.length; i++) {
|
||||
x[i] = (float) signal.at((double) i / RnnVad.SAMPLE_RATE);
|
||||
}
|
||||
return x;
|
||||
}
|
||||
|
||||
private static double[] run(float[] x) {
|
||||
RnnVad vad = new RnnVad();
|
||||
int frames = x.length / RnnVad.FRAME_SIZE;
|
||||
double[] probabilities = new double[frames];
|
||||
for (int i = 0; i < frames; i++) {
|
||||
probabilities[i] = vad.process(x, i * RnnVad.FRAME_SIZE);
|
||||
}
|
||||
return probabilities;
|
||||
}
|
||||
|
||||
private static double meanFrom(double[] values, int start) {
|
||||
double sum = 0;
|
||||
for (int i = start; i < values.length; i++) {
|
||||
sum += values[i];
|
||||
}
|
||||
return sum / (values.length - start);
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,12 @@ package com.ts3client.audio.desktop;
|
||||
|
||||
import com.github.manevolent.ts3j.enums.CodecType;
|
||||
import com.ts3client.audio.AudioEnhancer;
|
||||
import com.ts3client.audio.AudioFrameListener;
|
||||
import com.ts3client.audio.InputLevel;
|
||||
import com.ts3client.audio.OpusParameters;
|
||||
import com.ts3client.audio.SpeechDetector;
|
||||
import com.ts3client.audio.SpeechProbabilityDetector;
|
||||
import com.ts3client.audio.VoiceInput;
|
||||
import com.ts3client.audio.vad.RnnSpeechDetector;
|
||||
import com.ts3client.config.Settings;
|
||||
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
@@ -28,7 +31,23 @@ import java.util.function.Consumer;
|
||||
*/
|
||||
public final class DesktopVoiceInput implements VoiceInput {
|
||||
|
||||
private static final int HANGOVER_FRAMES = 15; // ~300 ms of tail after level drops
|
||||
/**
|
||||
* How long the gate stays open after the last active frame. TS3 counts 64 of its
|
||||
* 10 ms preprocessor frames before closing, so the tail of a word is never clipped.
|
||||
*/
|
||||
private static final int HANGOVER_MS = 640;
|
||||
|
||||
/**
|
||||
* Audio replayed when the gate opens, so a word's onset is not swallowed by the frame
|
||||
* that detected it. TS3 keeps up to three 10 ms buffers ({@code vad_extrabuffersize}
|
||||
* defaults to 2, plus one).
|
||||
*/
|
||||
private static final int PREROLL_MS = 30;
|
||||
|
||||
private static final int HANGOVER_FRAMES =
|
||||
Math.max(1, HANGOVER_MS * AudioDevices.SAMPLE_RATE / 1000 / AudioDevices.FRAME_SIZE);
|
||||
private static final int PREROLL_FRAMES =
|
||||
Math.max(1, PREROLL_MS * AudioDevices.SAMPLE_RATE / 1000 / AudioDevices.FRAME_SIZE);
|
||||
|
||||
private final ConcurrentLinkedQueue<byte[]> queue = new ConcurrentLinkedQueue<>();
|
||||
private final AtomicBoolean muted = new AtomicBoolean(false);
|
||||
@@ -44,12 +63,14 @@ public final class DesktopVoiceInput implements VoiceInput {
|
||||
private volatile double inputGain;
|
||||
private volatile CodecType codec = CodecType.OPUS_VOICE;
|
||||
|
||||
private final SpeechDetector speechDetector = new SpeechDetector(AudioDevices.SAMPLE_RATE);
|
||||
private final SpeechProbabilityDetector speechDetector =
|
||||
new RnnSpeechDetector(AudioDevices.SAMPLE_RATE);
|
||||
private final AudioEnhancer enhancer = new AudioEnhancer(AudioDevices.SAMPLE_RATE);
|
||||
|
||||
private volatile Consumer<Double> levelListener; // input level in dBFS
|
||||
private volatile Consumer<Double> levelListener; // input level, InputLevel scale
|
||||
private volatile Consumer<Boolean> talkListener; // local talk-state changes
|
||||
private volatile Runnable mutedTalkListener; // speech detected while muted
|
||||
private volatile AudioFrameListener monitorListener; // local monitoring of what is sent
|
||||
private boolean mutedTalking;
|
||||
|
||||
private final String deviceName;
|
||||
@@ -68,6 +89,11 @@ public final class DesktopVoiceInput implements VoiceInput {
|
||||
private int hangover;
|
||||
private boolean lastTransmitting;
|
||||
|
||||
// Ring of recent frames captured while the gate was shut.
|
||||
private final float[][] preroll = new float[PREROLL_FRAMES][];
|
||||
private int prerollTail;
|
||||
private int prerollCount;
|
||||
|
||||
public DesktopVoiceInput(Settings settings) {
|
||||
this.deviceName = settings.inputDevice;
|
||||
this.mode = settings.inputMode;
|
||||
@@ -195,6 +221,11 @@ public final class DesktopVoiceInput implements VoiceInput {
|
||||
this.talkListener = l;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setMonitorListener(AudioFrameListener l) {
|
||||
this.monitorListener = l;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setMutedTalkListener(Runnable l) {
|
||||
this.mutedTalkListener = l;
|
||||
@@ -305,32 +336,72 @@ public final class DesktopVoiceInput implements VoiceInput {
|
||||
// stream bypasses them and is transmitted as captured.
|
||||
if (!stereo) enhancer.process(mono, frameSamples);
|
||||
|
||||
double sumSq = 0;
|
||||
for (int i = 0; i < frameSamples; i++) {
|
||||
sumSq += (double) mono[i] * mono[i];
|
||||
}
|
||||
double rms = Math.sqrt(sumSq / frameSamples);
|
||||
double db = (rms <= 1e-9) ? -100.0 : 20.0 * Math.log10(rms);
|
||||
double db = InputLevel.toDb(mono, frameSamples);
|
||||
Consumer<Double> ll = levelListener;
|
||||
if (ll != null) ll.accept(db);
|
||||
|
||||
boolean wasOpen = transmitting.get();
|
||||
boolean open = decideGate(db, mono);
|
||||
setTransmitting(open);
|
||||
|
||||
if (open && !muted.get() && encoder != null) {
|
||||
try {
|
||||
packet = encoder.encode(stereo ? pcm : mono);
|
||||
// On the opening edge, send the buffered lead-in first so the
|
||||
// word's onset isn't lost to the frame that detected it.
|
||||
if (!wasOpen) {
|
||||
flushPreroll(stereo);
|
||||
}
|
||||
float[] sent = stereo ? pcm : mono;
|
||||
packet = encoder.encode(sent);
|
||||
monitor(sent, stereo ? encoderChannels : 1);
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
if (!open) {
|
||||
rememberForPreroll(stereo ? pcm : mono);
|
||||
}
|
||||
}
|
||||
|
||||
if (packet != null && packet.length > 0) {
|
||||
offer(packet);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void offer(byte[] packet) {
|
||||
queue.offer(packet);
|
||||
// Guard against unbounded growth if the network stalls.
|
||||
while (queue.size() > 10) queue.poll();
|
||||
}
|
||||
|
||||
/** Keeps the most recent frames while the gate is shut, for {@link #flushPreroll}. */
|
||||
private void rememberForPreroll(float[] frame) {
|
||||
float[] slot = preroll[prerollTail];
|
||||
if (slot == null || slot.length != frame.length) {
|
||||
slot = new float[frame.length];
|
||||
preroll[prerollTail] = slot;
|
||||
}
|
||||
System.arraycopy(frame, 0, slot, 0, frame.length);
|
||||
prerollTail = (prerollTail + 1) % PREROLL_FRAMES;
|
||||
if (prerollCount < PREROLL_FRAMES) prerollCount++;
|
||||
}
|
||||
|
||||
/** Encodes and queues the buffered lead-in, oldest first. Call under {@link #encoderLock}. */
|
||||
private void flushPreroll(boolean stereo) {
|
||||
int expected = stereo ? AudioDevices.FRAME_SIZE * encoderChannels : AudioDevices.FRAME_SIZE;
|
||||
for (int i = 0; i < prerollCount; i++) {
|
||||
int idx = (prerollTail - prerollCount + i + PREROLL_FRAMES) % PREROLL_FRAMES;
|
||||
float[] frame = preroll[idx];
|
||||
// A channel-count change between capture and flush invalidates the buffer.
|
||||
if (frame == null || frame.length != expected) continue;
|
||||
try {
|
||||
byte[] p = encoder.encode(frame);
|
||||
if (p != null && p.length > 0) offer(p);
|
||||
monitor(frame, stereo ? encoderChannels : 1);
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
prerollCount = 0;
|
||||
}
|
||||
|
||||
private boolean decideGate(double db, float[] pcm) {
|
||||
@@ -353,8 +424,10 @@ public final class DesktopVoiceInput implements VoiceInput {
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the selected VAD mode (volume gate, speech probability, or both) with
|
||||
* a deactivation-delay hangover so trailing syllables aren't clipped.
|
||||
* Applies the selected VAD mode with a hangover so trailing syllables aren't clipped.
|
||||
*
|
||||
* <p>The modes match the TS3 client's: the volume gate alone, the speech detector
|
||||
* alone, or both together. Volume Gate skips the detector entirely, as TS3 does.
|
||||
*/
|
||||
private boolean voiceActivated(double db, float[] pcm) {
|
||||
boolean detected;
|
||||
@@ -363,11 +436,11 @@ public final class DesktopVoiceInput implements VoiceInput {
|
||||
detected = db >= thresholdDb;
|
||||
break;
|
||||
case AUTOMATIC:
|
||||
detected = speechDetector.process(pcm) >= speechThreshold;
|
||||
detected = speechDetector.process(pcm, AudioDevices.FRAME_SIZE) >= speechThreshold;
|
||||
break;
|
||||
case HYBRID:
|
||||
default:
|
||||
double probability = speechDetector.process(pcm);
|
||||
double probability = speechDetector.process(pcm, AudioDevices.FRAME_SIZE);
|
||||
detected = db >= thresholdDb && probability >= speechThreshold;
|
||||
break;
|
||||
}
|
||||
@@ -395,6 +468,12 @@ public final class DesktopVoiceInput implements VoiceInput {
|
||||
mutedTalking = talking;
|
||||
}
|
||||
|
||||
/** Hands a transmitted frame to the monitor, if one is attached. */
|
||||
private void monitor(float[] frame, int channels) {
|
||||
AudioFrameListener l = monitorListener;
|
||||
if (l != null) l.onFrame(frame, channels);
|
||||
}
|
||||
|
||||
private void setTransmitting(boolean t) {
|
||||
transmitting.set(t);
|
||||
if (t != lastTransmitting) {
|
||||
|
||||
@@ -35,6 +35,12 @@
|
||||
<groupId>com.github.weisj</groupId>
|
||||
<artifactId>jsvg</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<version>5.13.4</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
||||
@@ -8,21 +8,28 @@ import java.awt.Graphics2D;
|
||||
import java.awt.RenderingHints;
|
||||
|
||||
/**
|
||||
* Horizontal audio level meter (dBFS) with an optional VAD threshold marker.
|
||||
* The filled portion turns green once the level crosses the threshold, giving
|
||||
* immediate visual feedback while tuning voice activation.
|
||||
* Horizontal audio level meter with an optional VAD threshold marker. The filled portion
|
||||
* turns green once the level crosses the threshold, giving immediate visual feedback while
|
||||
* tuning voice activation.
|
||||
*
|
||||
* <p>Scaled to {@link com.ts3client.audio.InputLevel}, matching the TS3 client's slider.
|
||||
*/
|
||||
public final class LevelMeter extends JComponent {
|
||||
|
||||
private static final double MIN_DB = -70.0;
|
||||
private static final double MAX_DB = 0.0;
|
||||
private static final double MIN_DB = com.ts3client.audio.InputLevel.MIN_DB;
|
||||
private static final double MAX_DB = com.ts3client.audio.InputLevel.MAX_DB;
|
||||
|
||||
private volatile double levelDb = MIN_DB;
|
||||
private volatile double thresholdDb = -45.0;
|
||||
private volatile double thresholdDb = -40.0;
|
||||
private volatile boolean showThreshold = true;
|
||||
private volatile boolean transmitting;
|
||||
|
||||
public LevelMeter() {
|
||||
setPreferredSize(new Dimension(240, 18));
|
||||
// A bare JComponent reports no minimum of its own, so a layout tight on space
|
||||
// collapses the bar to nothing. Keep the height and let only the width give.
|
||||
setMinimumSize(new Dimension(60, 18));
|
||||
setMaximumSize(new Dimension(Integer.MAX_VALUE, 18));
|
||||
}
|
||||
|
||||
public void setLevel(double db) {
|
||||
@@ -40,6 +47,16 @@ public final class LevelMeter extends JComponent {
|
||||
repaint();
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the gate is currently open. This is the real transmit decision, which in the
|
||||
* Automatic and Hybrid modes depends on the speech detector and the hangover as well as
|
||||
* on the level, so the bar cannot infer it from the threshold alone.
|
||||
*/
|
||||
public void setTransmitting(boolean transmitting) {
|
||||
this.transmitting = transmitting;
|
||||
repaint();
|
||||
}
|
||||
|
||||
private int dbToX(double db, int w) {
|
||||
double clamped = Math.max(MIN_DB, Math.min(MAX_DB, db));
|
||||
return (int) ((clamped - MIN_DB) / (MAX_DB - MIN_DB) * w);
|
||||
@@ -56,8 +73,7 @@ public final class LevelMeter extends JComponent {
|
||||
g.fillRoundRect(0, 0, w - 1, h - 1, 6, 6);
|
||||
|
||||
int level = dbToX(levelDb, w - 2);
|
||||
boolean over = levelDb >= thresholdDb;
|
||||
g.setColor(over ? Theme.TALKING : new Color(0x5A9BD4));
|
||||
g.setColor(transmitting ? Theme.TALKING : new Color(0x5A9BD4));
|
||||
g.fillRoundRect(1, 1, Math.max(0, level), h - 3, 5, 5);
|
||||
|
||||
if (showThreshold) {
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
package com.ts3client.ui;
|
||||
|
||||
import com.ts3client.audio.VoiceInput;
|
||||
import com.ts3client.audio.desktop.AudioDevices;
|
||||
import com.ts3client.audio.desktop.AudioPlayback;
|
||||
import com.ts3client.audio.desktop.DesktopVoiceInput;
|
||||
import com.ts3client.config.Settings;
|
||||
|
||||
import javax.swing.SwingUtilities;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
/**
|
||||
* Drives the settings dialog's microphone test from a real capture chain.
|
||||
*
|
||||
* <p>The dialog runs its own {@link DesktopVoiceInput} rather than borrowing the connected
|
||||
* one, whose listeners belong to the connection. Because it is the same class that feeds
|
||||
* the server, the level and the gate shown here are exactly what would be transmitted —
|
||||
* pre-processing, voice detection, hangover and pre-roll included.
|
||||
*
|
||||
* <p>Loopback is optional: when enabled, the frames that would be sent are played back
|
||||
* locally so you can hear precisely what the other side would.
|
||||
*/
|
||||
final class MicrophoneTest {
|
||||
|
||||
/** Frames buffered for loopback before the oldest is dropped (~100 ms). */
|
||||
private static final int LOOPBACK_QUEUE_FRAMES = 5;
|
||||
|
||||
private final Consumer<Double> onLevel;
|
||||
private final Consumer<Boolean> onTransmitting;
|
||||
|
||||
private DesktopVoiceInput mic;
|
||||
|
||||
private final ArrayBlockingQueue<byte[]> loopbackQueue =
|
||||
new ArrayBlockingQueue<>(LOOPBACK_QUEUE_FRAMES);
|
||||
private volatile boolean loopbackEnabled;
|
||||
private volatile boolean loopbackRunning;
|
||||
private Thread loopbackThread;
|
||||
private String outputDevice = "";
|
||||
|
||||
MicrophoneTest(Consumer<Double> onLevel, Consumer<Boolean> onTransmitting) {
|
||||
this.onLevel = onLevel;
|
||||
this.onTransmitting = onTransmitting;
|
||||
}
|
||||
|
||||
/** Whether a capture chain is currently running. */
|
||||
boolean isRunning() {
|
||||
return mic != null;
|
||||
}
|
||||
|
||||
/**
|
||||
* (Re)starts the test chain against {@code settings}' devices and voice options.
|
||||
*
|
||||
* @return false if the capture device could not be opened
|
||||
*/
|
||||
boolean start(Settings settings) {
|
||||
stop();
|
||||
DesktopVoiceInput input = new DesktopVoiceInput(settings);
|
||||
input.setLevelListener(db -> SwingUtilities.invokeLater(() -> onLevel.accept(db)));
|
||||
input.setTalkListener(talking -> SwingUtilities.invokeLater(() -> onTransmitting.accept(talking)));
|
||||
input.setMonitorListener(this::enqueueForLoopback);
|
||||
this.outputDevice = settings.outputDevice;
|
||||
try {
|
||||
input.start();
|
||||
} catch (RuntimeException e) {
|
||||
return false; // Device busy or gone; leave the test switched off.
|
||||
}
|
||||
this.mic = input;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops the capture chain. The caller owns the UI reset: doing it here would race with
|
||||
* a restart, whose own state has already been put on screen.
|
||||
*/
|
||||
void stop() {
|
||||
setLoopback(false);
|
||||
if (mic != null) {
|
||||
mic.stop();
|
||||
mic = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Applies a change to the test chain, if it is running. */
|
||||
void configure(Consumer<VoiceInput> change) {
|
||||
VoiceInput m = mic;
|
||||
if (m != null) change.accept(m);
|
||||
}
|
||||
|
||||
/** The playback device to loop back through; takes effect on the next enable. */
|
||||
void setOutputDevice(String deviceId) {
|
||||
this.outputDevice = deviceId == null ? "" : deviceId;
|
||||
}
|
||||
|
||||
void setLoopback(boolean enabled) {
|
||||
if (enabled == loopbackEnabled) {
|
||||
return;
|
||||
}
|
||||
loopbackEnabled = enabled;
|
||||
if (enabled) {
|
||||
loopbackRunning = true;
|
||||
loopbackThread = new Thread(this::loopbackLoop, "settings-loopback");
|
||||
loopbackThread.setDaemon(true);
|
||||
loopbackThread.start();
|
||||
} else {
|
||||
loopbackRunning = false;
|
||||
if (loopbackThread != null) {
|
||||
loopbackThread.interrupt();
|
||||
loopbackThread = null;
|
||||
}
|
||||
loopbackQueue.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a transmitted frame to 16-bit PCM and queues it. Runs on the capture thread,
|
||||
* so it must not block: a full queue means playback has fallen behind and the oldest
|
||||
* frame is dropped instead.
|
||||
*/
|
||||
private void enqueueForLoopback(float[] interleaved, int channels) {
|
||||
if (!loopbackEnabled) {
|
||||
return;
|
||||
}
|
||||
byte[] pcm = toPcm16(interleaved);
|
||||
if (!loopbackQueue.offer(pcm)) {
|
||||
loopbackQueue.poll();
|
||||
loopbackQueue.offer(pcm);
|
||||
}
|
||||
}
|
||||
|
||||
private void loopbackLoop() {
|
||||
AudioPlayback line = null;
|
||||
try {
|
||||
// The monitored frames are mono for voice; ask for a matching line so no
|
||||
// channel juggling is needed, and up-mix only if the device insists on stereo.
|
||||
line = AudioDevices.openPlayback(outputDevice, 1);
|
||||
line.start();
|
||||
int channels = line.channels();
|
||||
while (loopbackRunning) {
|
||||
byte[] frame = loopbackQueue.poll(100, java.util.concurrent.TimeUnit.MILLISECONDS);
|
||||
if (frame == null) {
|
||||
continue;
|
||||
}
|
||||
byte[] out = (channels == 1) ? frame : upmix(frame, channels);
|
||||
line.write(out, 0, out.length);
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
} catch (Exception ignored) {
|
||||
// The device may be busy or gone; the test simply runs without loopback.
|
||||
} finally {
|
||||
if (line != null) line.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** Converts float samples in [-1, 1] to 16-bit little-endian PCM. */
|
||||
static byte[] toPcm16(float[] samples) {
|
||||
byte[] pcm = new byte[samples.length * 2];
|
||||
for (int i = 0; i < samples.length; i++) {
|
||||
float f = samples[i];
|
||||
if (f > 1f) f = 1f;
|
||||
else if (f < -1f) f = -1f;
|
||||
int s = Math.round(f * 32767f);
|
||||
pcm[2 * i] = (byte) (s & 0xFF);
|
||||
pcm[2 * i + 1] = (byte) ((s >> 8) & 0xFF);
|
||||
}
|
||||
return pcm;
|
||||
}
|
||||
|
||||
/** Copies a mono frame across {@code channels} interleaved channels. */
|
||||
static byte[] upmix(byte[] mono, int channels) {
|
||||
byte[] out = new byte[mono.length * channels];
|
||||
for (int i = 0, frames = mono.length / 2; i < frames; i++) {
|
||||
for (int c = 0; c < channels; c++) {
|
||||
out[2 * (i * channels + c)] = mono[2 * i];
|
||||
out[2 * (i * channels + c) + 1] = mono[2 * i + 1];
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
package com.ts3client.ui;
|
||||
|
||||
import com.ts3client.audio.InputLevel;
|
||||
import com.ts3client.audio.OpusParameters;
|
||||
import com.ts3client.audio.VoiceInput;
|
||||
import com.ts3client.audio.VoiceOutput;
|
||||
import com.ts3client.audio.desktop.AudioCapture;
|
||||
import com.ts3client.audio.desktop.AudioDevices;
|
||||
import com.ts3client.config.Settings;
|
||||
import com.ts3client.hotkey.Hotkey;
|
||||
@@ -23,6 +23,7 @@ import javax.swing.JPanel;
|
||||
import javax.swing.JRadioButton;
|
||||
import javax.swing.JSlider;
|
||||
import javax.swing.JTabbedPane;
|
||||
import javax.swing.JToggleButton;
|
||||
import javax.swing.Scrollable;
|
||||
import javax.swing.SwingUtilities;
|
||||
import java.awt.BorderLayout;
|
||||
@@ -80,6 +81,9 @@ public final class SettingsDialog extends JDialog {
|
||||
private JLabel thresholdLabel;
|
||||
private JLabel speechLabel;
|
||||
private LevelMeter meter;
|
||||
private JToggleButton testButton;
|
||||
private JCheckBox loopbackCheck;
|
||||
private JLabel talkIndicator;
|
||||
private JButton pttKeyButton;
|
||||
private JSlider bitrateSlider;
|
||||
private JLabel bitrateLabel;
|
||||
@@ -88,8 +92,7 @@ public final class SettingsDialog extends JDialog {
|
||||
private JCheckBox fecCheck;
|
||||
private JCheckBox musicCheck;
|
||||
|
||||
private volatile boolean meterRunning;
|
||||
private Thread meterThread;
|
||||
private final MicrophoneTest micTest = new MicrophoneTest(this::onTestLevel, this::onTestTalking);
|
||||
|
||||
public SettingsDialog(Frame owner, Settings settings,
|
||||
VoiceInput liveMic, VoiceOutput livePlayback,
|
||||
@@ -136,7 +139,7 @@ public final class SettingsDialog extends JDialog {
|
||||
addWindowListener(new java.awt.event.WindowAdapter() {
|
||||
@Override
|
||||
public void windowClosed(java.awt.event.WindowEvent e) {
|
||||
stopMeter();
|
||||
micTest.stop();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -144,7 +147,6 @@ public final class SettingsDialog extends JDialog {
|
||||
setSize(new Dimension(480, 540));
|
||||
setMinimumSize(new Dimension(420, 360));
|
||||
setLocationRelativeTo(owner);
|
||||
startMeter();
|
||||
}
|
||||
|
||||
private JPanel buildDevicesTab() {
|
||||
@@ -181,10 +183,10 @@ public final class SettingsDialog extends JDialog {
|
||||
outputVol.addChangeListener(e -> {
|
||||
if (livePlayback != null) livePlayback.setMasterVolume(outputVol.getValue() / 100.0);
|
||||
});
|
||||
inputGain.addChangeListener(e -> {
|
||||
if (liveMic != null) liveMic.setInputGain(inputGain.getValue() / 100.0);
|
||||
});
|
||||
inputCombo.addActionListener(e -> restartMeter());
|
||||
inputGain.addChangeListener(e ->
|
||||
applyLive(m -> m.setInputGain(inputGain.getValue() / 100.0)));
|
||||
inputCombo.addActionListener(e -> restartTest());
|
||||
outputCombo.addActionListener(e -> micTest.setOutputDevice(comboValue(outputCombo)));
|
||||
|
||||
c.gridx = 0;
|
||||
c.gridy = row++;
|
||||
@@ -223,19 +225,18 @@ public final class SettingsDialog extends JDialog {
|
||||
|
||||
Runnable syncNoise = () -> {
|
||||
denoiseLevel.setEnabled(denoiseCheck.isSelected());
|
||||
if (liveMic != null) {
|
||||
liveMic.setNoiseSuppression(denoiseCheck.isSelected());
|
||||
liveMic.setDenoiserLevel(denoiseLevel.getValue() / 100.0);
|
||||
liveMic.setTypingAttenuation(typingCheck.isSelected());
|
||||
liveMic.setAgc(agcCheck.isSelected());
|
||||
}
|
||||
applyLive(m -> {
|
||||
m.setNoiseSuppression(denoiseCheck.isSelected());
|
||||
m.setDenoiserLevel(denoiseLevel.getValue() / 100.0);
|
||||
m.setTypingAttenuation(typingCheck.isSelected());
|
||||
m.setAgc(agcCheck.isSelected());
|
||||
});
|
||||
};
|
||||
denoiseCheck.addActionListener(e -> syncNoise.run());
|
||||
typingCheck.addActionListener(e -> syncNoise.run());
|
||||
agcCheck.addActionListener(e -> syncNoise.run());
|
||||
denoiseLevel.addChangeListener(e -> {
|
||||
if (liveMic != null) liveMic.setDenoiserLevel(denoiseLevel.getValue() / 100.0);
|
||||
});
|
||||
denoiseLevel.addChangeListener(e ->
|
||||
applyLive(m -> m.setDenoiserLevel(denoiseLevel.getValue() / 100.0)));
|
||||
syncNoise.run();
|
||||
|
||||
// filler
|
||||
@@ -285,26 +286,30 @@ public final class SettingsDialog extends JDialog {
|
||||
c.gridy = row;
|
||||
c.gridwidth = 2;
|
||||
c.insets = new Insets(10, 4, 2, 4);
|
||||
p.add(new JLabel("Input level (speak to test):"), c);
|
||||
p.add(new JLabel("Input level:"), c);
|
||||
c.gridy = ++row;
|
||||
p.add(meter, c);
|
||||
c.insets = new Insets(4, 4, 4, 4);
|
||||
c.gridy = ++row;
|
||||
p.add(buildTestControls(), c);
|
||||
c.gridwidth = 1;
|
||||
row++;
|
||||
|
||||
vadModeCombo = new JComboBox<>(new String[]{"Automatic", "Volume Gate", "Hybrid"});
|
||||
vadModeCombo.setSelectedIndex(vadModeIndex(settings.vadMode));
|
||||
vadModeCombo.addActionListener(e -> micTest.configure(m -> m.setVadMode(currentVadMode())));
|
||||
vadModeCombo.setToolTipText("<html><b>Automatic</b>: intelligent speech detection.<br>"
|
||||
+ "<b>Volume Gate</b>: transmit when loud enough.<br>"
|
||||
+ "<b>Hybrid</b>: loud enough <i>and</i> detected as speech.</html>");
|
||||
addRow(p, c, row++, new JLabel("Detection:"), vadModeCombo);
|
||||
|
||||
thresholdSlider = new JSlider(-70, 0, (int) Math.round(settings.vadThresholdDb));
|
||||
thresholdSlider = new JSlider((int) InputLevel.MIN_DB, (int) InputLevel.MAX_DB,
|
||||
(int) Math.round(settings.vadThresholdDb));
|
||||
thresholdLabel = new JLabel(Math.round(settings.vadThresholdDb) + " dB");
|
||||
thresholdSlider.addChangeListener(e -> {
|
||||
meter.setThreshold(thresholdSlider.getValue());
|
||||
thresholdLabel.setText(thresholdSlider.getValue() + " dB");
|
||||
if (liveMic != null) liveMic.setThresholdDb(thresholdSlider.getValue());
|
||||
applyLive(m -> m.setThresholdDb(thresholdSlider.getValue()));
|
||||
});
|
||||
addRow(p, c, row++, new JLabel("Volume gate:"), sliderWithLabel(thresholdSlider, thresholdLabel));
|
||||
|
||||
@@ -312,7 +317,7 @@ public final class SettingsDialog extends JDialog {
|
||||
speechLabel = new JLabel(Math.round(settings.speechThreshold * 100) + "%");
|
||||
speechSlider.addChangeListener(e -> {
|
||||
speechLabel.setText(speechSlider.getValue() + "%");
|
||||
if (liveMic != null) liveMic.setSpeechThreshold(speechSlider.getValue() / 100.0);
|
||||
applyLive(m -> m.setSpeechThreshold(speechSlider.getValue() / 100.0));
|
||||
});
|
||||
addRow(p, c, row++, new JLabel("Speech threshold:"), sliderWithLabel(speechSlider, speechLabel));
|
||||
|
||||
@@ -322,6 +327,8 @@ public final class SettingsDialog extends JDialog {
|
||||
addRow(p, c, row++, new JLabel("Push-to-talk hotkey:"), pttKeyButton);
|
||||
|
||||
vadOverPttCheck = new JCheckBox("Also detect voice while push-to-talk", settings.vadOverPtt);
|
||||
vadOverPttCheck.addActionListener(e ->
|
||||
micTest.configure(m -> m.setVadOverPtt(vadOverPttCheck.isSelected())));
|
||||
c.gridx = 0;
|
||||
c.gridy = row++;
|
||||
c.gridwidth = 2;
|
||||
@@ -542,46 +549,67 @@ public final class SettingsDialog extends JDialog {
|
||||
hotkeysPanel.reload();
|
||||
}
|
||||
|
||||
private void apply() {
|
||||
settings.inputDevice = comboValue(inputCombo);
|
||||
settings.outputDevice = comboValue(outputCombo);
|
||||
settings.inputVolume = inputGain.getValue() / 100.0;
|
||||
settings.outputVolume = outputVol.getValue() / 100.0;
|
||||
settings.denoise = denoiseCheck.isSelected();
|
||||
settings.denoiserLevel = denoiseLevel.getValue() / 100.0;
|
||||
settings.typingAttenuation = typingCheck.isSelected();
|
||||
settings.agc = agcCheck.isSelected();
|
||||
settings.inputMode = pttRadio.isSelected() ? Settings.InputMode.PUSH_TO_TALK
|
||||
/** Copies the audio form into {@code target}, without touching anything else. */
|
||||
private void writeAudioSettings(Settings target) {
|
||||
target.inputDevice = comboValue(inputCombo);
|
||||
target.outputDevice = comboValue(outputCombo);
|
||||
target.inputVolume = inputGain.getValue() / 100.0;
|
||||
target.outputVolume = outputVol.getValue() / 100.0;
|
||||
target.denoise = denoiseCheck.isSelected();
|
||||
target.denoiserLevel = denoiseLevel.getValue() / 100.0;
|
||||
target.typingAttenuation = typingCheck.isSelected();
|
||||
target.agc = agcCheck.isSelected();
|
||||
target.inputMode = pttRadio.isSelected() ? Settings.InputMode.PUSH_TO_TALK
|
||||
: contRadio.isSelected() ? Settings.InputMode.CONTINUOUS
|
||||
: Settings.InputMode.VOICE_ACTIVATION;
|
||||
settings.vadMode = currentVadMode();
|
||||
settings.vadThresholdDb = thresholdSlider.getValue();
|
||||
settings.speechThreshold = speechSlider.getValue() / 100.0;
|
||||
settings.vadOverPtt = vadOverPttCheck.isSelected();
|
||||
settings.bitrate = bitrateSlider.getValue() * 1000;
|
||||
settings.complexity = complexitySlider.getValue();
|
||||
settings.vbr = vbrCheck.isSelected();
|
||||
settings.fec = fecCheck.isSelected();
|
||||
settings.music = musicCheck.isSelected();
|
||||
target.vadMode = currentVadMode();
|
||||
target.vadThresholdDb = thresholdSlider.getValue();
|
||||
target.speechThreshold = speechSlider.getValue() / 100.0;
|
||||
target.vadOverPtt = vadOverPttCheck.isSelected();
|
||||
target.bitrate = bitrateSlider.getValue() * 1000;
|
||||
target.complexity = complexitySlider.getValue();
|
||||
target.vbr = vbrCheck.isSelected();
|
||||
target.fec = fecCheck.isSelected();
|
||||
target.music = musicCheck.isSelected();
|
||||
}
|
||||
|
||||
/**
|
||||
* The audio form as a standalone {@link Settings}, so the test chain runs with the
|
||||
* values currently on screen rather than the ones last saved.
|
||||
*
|
||||
* <p>The test always runs voice activation: it exists to tune the gate, and push-to-talk
|
||||
* would need the global hotkey, which belongs to the connected microphone.
|
||||
*/
|
||||
private Settings audioSnapshot() {
|
||||
Settings snapshot = new Settings();
|
||||
writeAudioSettings(snapshot);
|
||||
if (snapshot.inputMode == Settings.InputMode.PUSH_TO_TALK) {
|
||||
snapshot.inputMode = Settings.InputMode.VOICE_ACTIVATION;
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
private void apply() {
|
||||
writeAudioSettings(settings);
|
||||
notificationsPanel.apply();
|
||||
iconPackPanel.apply();
|
||||
clientVersionPanel.apply();
|
||||
hotkeysPanel.apply();
|
||||
settings.save();
|
||||
|
||||
if (liveMic != null) {
|
||||
liveMic.setMode(settings.inputMode);
|
||||
liveMic.setVadMode(settings.vadMode);
|
||||
liveMic.setThresholdDb(settings.vadThresholdDb);
|
||||
liveMic.setSpeechThreshold(settings.speechThreshold);
|
||||
liveMic.setVadOverPtt(settings.vadOverPtt);
|
||||
liveMic.setInputGain(settings.inputVolume);
|
||||
liveMic.setNoiseSuppression(settings.denoise);
|
||||
liveMic.setDenoiserLevel(settings.denoiserLevel);
|
||||
liveMic.setTypingAttenuation(settings.typingAttenuation);
|
||||
liveMic.setAgc(settings.agc);
|
||||
liveMic.setOpusParameters(OpusParameters.from(settings));
|
||||
}
|
||||
applyLive(m -> {
|
||||
m.setMode(settings.inputMode);
|
||||
m.setVadMode(settings.vadMode);
|
||||
m.setThresholdDb(settings.vadThresholdDb);
|
||||
m.setSpeechThreshold(settings.speechThreshold);
|
||||
m.setVadOverPtt(settings.vadOverPtt);
|
||||
m.setInputGain(settings.inputVolume);
|
||||
m.setNoiseSuppression(settings.denoise);
|
||||
m.setDenoiserLevel(settings.denoiserLevel);
|
||||
m.setTypingAttenuation(settings.typingAttenuation);
|
||||
m.setAgc(settings.agc);
|
||||
m.setOpusParameters(OpusParameters.from(settings));
|
||||
});
|
||||
if (livePlayback != null) {
|
||||
livePlayback.setMasterVolume(settings.outputVolume);
|
||||
livePlayback.setOutputDevice(settings.outputDevice);
|
||||
@@ -597,64 +625,91 @@ public final class SettingsDialog extends JDialog {
|
||||
}
|
||||
|
||||
private void close() {
|
||||
stopMeter();
|
||||
micTest.stop();
|
||||
dispose();
|
||||
}
|
||||
|
||||
// ---- live meter ----
|
||||
// ---- microphone test ----
|
||||
|
||||
private void startMeter() {
|
||||
meterRunning = true;
|
||||
meterThread = new Thread(this::meterLoop, "settings-meter");
|
||||
meterThread.setDaemon(true);
|
||||
meterThread.start();
|
||||
/**
|
||||
* The test row: a toggle that runs the capture chain, an indicator showing whether the
|
||||
* gate is open, and an optional loopback so you can hear what is being sent.
|
||||
*/
|
||||
private JPanel buildTestControls() {
|
||||
JPanel row = new JPanel(new java.awt.FlowLayout(java.awt.FlowLayout.LEFT, 8, 0));
|
||||
|
||||
testButton = new JToggleButton("Begin Test");
|
||||
testButton.setToolTipText("Run the capture chain exactly as it runs while connected, "
|
||||
+ "so the bar and the indicator show what would actually be transmitted.");
|
||||
testButton.addActionListener(e -> setTesting(testButton.isSelected()));
|
||||
|
||||
loopbackCheck = new JCheckBox("Hear myself");
|
||||
loopbackCheck.setToolTipText("Play the transmitted audio back through the playback "
|
||||
+ "device. Use headphones to avoid feedback.");
|
||||
loopbackCheck.setEnabled(false);
|
||||
loopbackCheck.addActionListener(e -> micTest.setLoopback(loopbackCheck.isSelected()));
|
||||
|
||||
talkIndicator = new JLabel("Not transmitting", Icons.clientIdle(), JLabel.LEFT);
|
||||
talkIndicator.setToolTipText("Lights up while your microphone is open, exactly as "
|
||||
+ "other users would see you in the channel list.");
|
||||
|
||||
row.add(testButton);
|
||||
row.add(loopbackCheck);
|
||||
row.add(talkIndicator);
|
||||
return row;
|
||||
}
|
||||
|
||||
private void restartMeter() {
|
||||
stopMeter();
|
||||
startMeter();
|
||||
private void setTesting(boolean on) {
|
||||
if (on && !micTest.start(audioSnapshot())) {
|
||||
testButton.setSelected(false);
|
||||
testButton.setText("Begin Test");
|
||||
loopbackCheck.setEnabled(false);
|
||||
resetTestIndicators();
|
||||
talkIndicator.setText("Capture device unavailable");
|
||||
return;
|
||||
}
|
||||
if (!on) {
|
||||
micTest.stop();
|
||||
loopbackCheck.setSelected(false);
|
||||
resetTestIndicators();
|
||||
}
|
||||
testButton.setText(on ? "Stop Test" : "Begin Test");
|
||||
loopbackCheck.setEnabled(on);
|
||||
}
|
||||
|
||||
private void stopMeter() {
|
||||
meterRunning = false;
|
||||
if (meterThread != null) {
|
||||
meterThread.interrupt();
|
||||
meterThread = null;
|
||||
private void resetTestIndicators() {
|
||||
onTestTalking(false);
|
||||
onTestLevel(InputLevel.SILENCE_DB);
|
||||
}
|
||||
|
||||
/** Restarts the test chain, if running, so a device change takes effect. */
|
||||
private void restartTest() {
|
||||
if (micTest.isRunning()) {
|
||||
boolean loopback = loopbackCheck.isSelected();
|
||||
if (micTest.start(audioSnapshot())) {
|
||||
micTest.setLoopback(loopback);
|
||||
} else {
|
||||
setTesting(false);
|
||||
testButton.setSelected(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void meterLoop() {
|
||||
String device = comboValue(inputCombo);
|
||||
AudioCapture line = null;
|
||||
try {
|
||||
line = AudioDevices.openCapture(device, AudioDevices.MAX_CHANNELS);
|
||||
line.start();
|
||||
int frame = AudioDevices.FRAME_SIZE;
|
||||
int channels = line.channels();
|
||||
byte[] buf = new byte[frame * 2 * channels];
|
||||
double gain = (inputGain != null ? inputGain.getValue() / 100.0 : 1.0);
|
||||
while (meterRunning) {
|
||||
if (line.read(buf, 0, buf.length) < buf.length) break;
|
||||
// Metering follows the capture chain: level is taken off the downmix.
|
||||
double sumSq = 0;
|
||||
for (int i = 0; i < frame; i++) {
|
||||
double mono = 0;
|
||||
for (int c = 0; c < channels; c++) {
|
||||
int k = i * channels + c;
|
||||
short s = (short) ((buf[2 * k + 1] << 8) | (buf[2 * k] & 0xFF));
|
||||
mono += s / 32768.0;
|
||||
/** Applies a live change to the connected microphone and to the test one alike. */
|
||||
private void applyLive(java.util.function.Consumer<VoiceInput> change) {
|
||||
if (liveMic != null) change.accept(liveMic);
|
||||
micTest.configure(change);
|
||||
}
|
||||
double f = mono / channels * gain;
|
||||
sumSq += f * f;
|
||||
|
||||
private void onTestLevel(double db) {
|
||||
if (meter != null) meter.setLevel(db);
|
||||
}
|
||||
double rms = Math.sqrt(sumSq / frame);
|
||||
double db = rms <= 1e-9 ? -100 : 20 * Math.log10(rms);
|
||||
final double fdb = db;
|
||||
if (meter != null) SwingUtilities.invokeLater(() -> meter.setLevel(fdb));
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
} finally {
|
||||
if (line != null) line.close();
|
||||
|
||||
private void onTestTalking(boolean talking) {
|
||||
if (meter != null) meter.setTransmitting(talking);
|
||||
if (talkIndicator != null) {
|
||||
talkIndicator.setIcon(talking ? Icons.clientTalking() : Icons.clientIdle());
|
||||
talkIndicator.setText(talking ? "Transmitting" : "Not transmitting");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package com.ts3client.ui;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class MicrophoneTestTest {
|
||||
|
||||
private static int sampleAt(byte[] pcm, int index) {
|
||||
return (short) (((pcm[2 * index + 1] & 0xFF) << 8) | (pcm[2 * index] & 0xFF));
|
||||
}
|
||||
|
||||
@Test
|
||||
void convertsFloatSamplesToLittleEndianPcm() {
|
||||
byte[] pcm = MicrophoneTest.toPcm16(new float[]{0f, 1f, -1f, 0.5f});
|
||||
assertEquals(8, pcm.length);
|
||||
assertEquals(0, sampleAt(pcm, 0));
|
||||
assertEquals(32767, sampleAt(pcm, 1));
|
||||
assertEquals(-32767, sampleAt(pcm, 2));
|
||||
assertEquals(16384, sampleAt(pcm, 3), 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void clampsSamplesOutsideRange() {
|
||||
byte[] pcm = MicrophoneTest.toPcm16(new float[]{2f, -2f});
|
||||
assertEquals(32767, sampleAt(pcm, 0));
|
||||
assertEquals(-32767, sampleAt(pcm, 1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void upmixRepeatsMonoAcrossChannels() {
|
||||
byte[] mono = MicrophoneTest.toPcm16(new float[]{0.25f, -0.25f});
|
||||
byte[] stereo = MicrophoneTest.upmix(mono, 2);
|
||||
assertEquals(mono.length * 2, stereo.length);
|
||||
// Each mono sample appears once per channel, in order.
|
||||
assertEquals(sampleAt(mono, 0), sampleAt(stereo, 0));
|
||||
assertEquals(sampleAt(mono, 0), sampleAt(stereo, 1));
|
||||
assertEquals(sampleAt(mono, 1), sampleAt(stereo, 2));
|
||||
assertEquals(sampleAt(mono, 1), sampleAt(stereo, 3));
|
||||
}
|
||||
|
||||
@Test
|
||||
void upmixIsIdentityForOneChannel() {
|
||||
byte[] mono = MicrophoneTest.toPcm16(new float[]{0.1f, 0.2f, 0.3f});
|
||||
assertArrayEquals(mono, MicrophoneTest.upmix(mono, 1));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user