Move the voice pipelines into core
Capture processing, the jitter buffer and decoding were desktop classes, so any other platform would have had to copy them. They now live in core and reach the platform only through two interfaces: - AudioIo lists devices and opens capture/playback lines; the desktop's AudioDevices implements it over PipeWire and Java Sound. - OpusCodec creates encoders and decoders; the desktop binds libopus through the FFM API as before. DesktopVoiceInput becomes CaptureVoiceInput unchanged in behaviour. DesktopVoiceOutput splits into VoiceStream, one speaker's jitter buffer and decoder, paced by whoever pulls it, and StreamingVoiceOutput around it. Speakers reach the device either on a line each (the desktop, so each shows up in the PipeWire mixer) or mixed onto one shared line, as mobile audio APIs want. The settings dialog's device lists and microphone test now go through the AudioBackend instead of desktop classes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,53 @@
|
||||
package com.ts3client.audio;
|
||||
|
||||
import com.ts3client.audio.opus.OpusCodec;
|
||||
import com.ts3client.audio.opus.OpusDecoder;
|
||||
import com.ts3client.audio.opus.OpusEncoder;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* Stands in for libopus: a "packet" is one byte, decoded to a 20 ms frame holding that byte
|
||||
* divided by 100 in every sample. Concealment decodes to -1, so it is easy to spot.
|
||||
*/
|
||||
final class FakeOpus implements OpusCodec {
|
||||
|
||||
static final float CONCEALED = -1f;
|
||||
|
||||
int resets;
|
||||
|
||||
static byte[] packet(int value) {
|
||||
return new byte[]{(byte) value};
|
||||
}
|
||||
|
||||
@Override
|
||||
public OpusEncoder createEncoder(int sampleRate, int frameSize, int channels, Application application) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public OpusDecoder createDecoder(int sampleRate, int maxFrameSize, int channels) {
|
||||
return new OpusDecoder() {
|
||||
@Override
|
||||
public int decode(byte[] packet, float[] out) {
|
||||
float v = packet == null ? CONCEALED : packet[0] / 100f;
|
||||
Arrays.fill(out, 0, VoiceFormat.FRAME_SIZE * channels, v);
|
||||
return VoiceFormat.FRAME_SIZE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reset() {
|
||||
resets++;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public String version() {
|
||||
return "fake";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package com.ts3client.audio;
|
||||
|
||||
import com.github.manevolent.ts3j.enums.CodecType;
|
||||
import com.github.manevolent.ts3j.protocol.ProtocolRole;
|
||||
import com.github.manevolent.ts3j.protocol.packet.PacketBody0Voice;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class MixedPlayoutTest {
|
||||
|
||||
/** A mono line that records every frame written and blocks until the test lets it go on. */
|
||||
private static final class RecordingLine implements AudioPlayback {
|
||||
final BlockingQueue<Short> firstSamples = new LinkedBlockingQueue<>();
|
||||
final Semaphore permits = new Semaphore(0);
|
||||
|
||||
@Override
|
||||
public int channels() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(byte[] buffer, int offset, int length) {
|
||||
firstSamples.add((short) ((buffer[offset + 1] << 8) | (buffer[offset] & 0xFF)));
|
||||
permits.acquireUninterruptibly();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void drain() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void mixesSimultaneousSpeakersOntoOneLine() throws Exception {
|
||||
RecordingLine line = new RecordingLine();
|
||||
AudioIo io = new AudioIo() {
|
||||
@Override
|
||||
public List<AudioDevice> inputDevices() {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioDevice> outputDevices() {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
public AudioCapture openCapture(String deviceId, int preferredChannels) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public AudioPlayback openPlayback(String deviceId, int preferredChannels) {
|
||||
return line;
|
||||
}
|
||||
};
|
||||
StreamingVoiceOutput output = new StreamingVoiceOutput(io, new FakeOpus(), "", StreamingVoiceOutput.Lines.SHARED);
|
||||
try {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
output.handleVoice(voice(1, i, 25));
|
||||
output.handleVoice(voice(2, i, 50));
|
||||
}
|
||||
line.permits.release(100);
|
||||
|
||||
short expected = (short) Math.round(0.75f * 32767f);
|
||||
boolean mixed = false;
|
||||
for (int i = 0; i < 20 && !mixed; i++) {
|
||||
Short sample = line.firstSamples.poll(1, TimeUnit.SECONDS);
|
||||
assertNotNull(sample, "the mixer stopped writing");
|
||||
mixed = sample == expected;
|
||||
}
|
||||
assertTrue(mixed, "no frame carried both speakers");
|
||||
} finally {
|
||||
output.shutdown();
|
||||
line.permits.release(1000);
|
||||
}
|
||||
}
|
||||
|
||||
private static PacketBody0Voice voice(int clientId, int packetId, int value) {
|
||||
PacketBody0Voice voice = new PacketBody0Voice(ProtocolRole.SERVER);
|
||||
voice.setClientId(clientId);
|
||||
voice.setPacketId(packetId);
|
||||
voice.setCodecType(CodecType.OPUS_VOICE);
|
||||
voice.setCodecData(FakeOpus.packet(value));
|
||||
return voice;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package com.ts3client.audio;
|
||||
|
||||
import com.github.manevolent.ts3j.enums.CodecType;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class VoiceStreamTest {
|
||||
|
||||
private final FakeOpus opus = new FakeOpus();
|
||||
private final List<Boolean> talk = new ArrayList<>();
|
||||
private final VoiceStream stream = new VoiceStream(7, opus, (id, talking) -> talk.add(talking));
|
||||
private final float[] out = new float[VoiceStream.MAX_FRAME * 2];
|
||||
|
||||
private boolean offer(int packetId, int value) {
|
||||
return stream.offer(packetId, CodecType.OPUS_VOICE, FakeOpus.packet(value));
|
||||
}
|
||||
|
||||
/** Pulls one frame and returns its first sample, or NaN if nothing was played. */
|
||||
private float pull() {
|
||||
int frames = stream.pull(out, 1, 1.0);
|
||||
return frames > 0 ? out[0] : Float.NaN;
|
||||
}
|
||||
|
||||
@Test
|
||||
void holdsBackTheStartOfABurstThenPlaysInOrder() {
|
||||
assertTrue(offer(10, 1));
|
||||
assertFalse(offer(12, 3));
|
||||
assertFalse(offer(11, 2));
|
||||
|
||||
assertEquals(Float.NaN, pull());
|
||||
assertEquals(Float.NaN, pull());
|
||||
assertEquals(0.01f, pull());
|
||||
assertEquals(0.02f, pull());
|
||||
assertEquals(0.03f, pull());
|
||||
assertEquals(List.of(true), talk);
|
||||
}
|
||||
|
||||
@Test
|
||||
void concealsALostPacket() {
|
||||
offer(0, 1);
|
||||
offer(2, 3);
|
||||
pull();
|
||||
pull();
|
||||
assertEquals(0.01f, pull());
|
||||
assertEquals(FakeOpus.CONCEALED, pull());
|
||||
assertEquals(0.03f, pull());
|
||||
}
|
||||
|
||||
@Test
|
||||
void dropsPacketsThatArriveTooLate() {
|
||||
offer(0, 1);
|
||||
offer(1, 2);
|
||||
pull();
|
||||
pull();
|
||||
pull();
|
||||
pull();
|
||||
assertFalse(offer(0, 9));
|
||||
assertEquals(FakeOpus.CONCEALED, pull());
|
||||
}
|
||||
|
||||
@Test
|
||||
void endPacketFinishesTheBurst() {
|
||||
offer(0, 1);
|
||||
stream.offer(1, CodecType.OPUS_VOICE, new byte[0]);
|
||||
pull();
|
||||
pull();
|
||||
pull();
|
||||
assertEquals(VoiceStream.IDLE, stream.pull(out, 1, 1.0));
|
||||
assertFalse(stream.isPlaying());
|
||||
assertEquals(List.of(true, false), talk);
|
||||
assertEquals(1, opus.resets);
|
||||
assertTrue(offer(2, 1), "the next packet starts a new burst");
|
||||
}
|
||||
|
||||
@Test
|
||||
void givesUpAfterTooMuchConcealment() {
|
||||
offer(0, 1);
|
||||
pull();
|
||||
pull();
|
||||
pull();
|
||||
int concealed = 0;
|
||||
while (stream.pull(out, 1, 1.0) != VoiceStream.IDLE) concealed++;
|
||||
assertEquals(10, concealed);
|
||||
assertEquals(List.of(true, false), talk);
|
||||
}
|
||||
|
||||
@Test
|
||||
void stopFromAnotherThreadEndsTheBurst() {
|
||||
offer(0, 1);
|
||||
stream.stop();
|
||||
assertEquals(VoiceStream.IDLE, stream.pull(out, 1, 1.0));
|
||||
assertEquals(List.of(true, false), talk);
|
||||
}
|
||||
|
||||
@Test
|
||||
void spreadsMonoOverStereoWithGain() {
|
||||
offer(0, 50);
|
||||
stream.pull(out, 2, 0.5);
|
||||
stream.pull(out, 2, 0.5);
|
||||
assertEquals(VoiceFormat.FRAME_SIZE, stream.pull(out, 2, 0.5));
|
||||
assertEquals(0.25f, out[0]);
|
||||
assertEquals(0.25f, out[1]);
|
||||
assertEquals(0.25f, out[2 * VoiceFormat.FRAME_SIZE - 1]);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user