Add voice recording
Tools > Start Recording / Start Multitrack Recording / Stop Recording record what is heard on the server, our own voice included, into Ogg Opus, and publish client_is_recording. A plain recording mixes everyone down to mono; a multitrack one gives each speaker (by unique id) an uncoupled mono stream of their own under channel mapping family 255, named in the tags after their nickname when the recording started, silent until they first speak, and continuing into name-2.ogg past 255 speakers. Tracks are spooled beside the output and muxed on stop, since the header carries the channel count. The status bar shows a label that stops the recording; the existing recording hotkeys start one straight into the last-used folder. OggOpusFile now shares the new Ogg page writer and Opus header code. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,59 @@
|
||||
package com.ts3client.recording;
|
||||
|
||||
import java.lang.foreign.Arena;
|
||||
import java.lang.foreign.FunctionDescriptor;
|
||||
import java.lang.foreign.Linker;
|
||||
import java.lang.foreign.MemorySegment;
|
||||
import java.lang.foreign.SymbolLookup;
|
||||
import java.lang.foreign.ValueLayout;
|
||||
import java.lang.invoke.MethodHandle;
|
||||
|
||||
/** libopus' own multistream decoder, to check our multistream packets against. */
|
||||
final class LibOpusMultistream implements AutoCloseable {
|
||||
|
||||
private static final Linker LINKER = Linker.nativeLinker();
|
||||
private static final SymbolLookup LIB = SymbolLookup.libraryLookup("libopus.so.0", Arena.global());
|
||||
private static final MethodHandle CREATE = LINKER.downcallHandle(LIB.find("opus_multistream_decoder_create").orElseThrow(),
|
||||
FunctionDescriptor.of(ValueLayout.ADDRESS, ValueLayout.JAVA_INT, ValueLayout.JAVA_INT, ValueLayout.JAVA_INT,
|
||||
ValueLayout.JAVA_INT, ValueLayout.ADDRESS, ValueLayout.ADDRESS));
|
||||
private static final MethodHandle DECODE = LINKER.downcallHandle(LIB.find("opus_multistream_decode_float").orElseThrow(),
|
||||
FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.ADDRESS, ValueLayout.ADDRESS, ValueLayout.JAVA_INT,
|
||||
ValueLayout.ADDRESS, ValueLayout.JAVA_INT, ValueLayout.JAVA_INT));
|
||||
private static final MethodHandle DESTROY = LINKER.downcallHandle(LIB.find("opus_multistream_decoder_destroy").orElseThrow(),
|
||||
FunctionDescriptor.ofVoid(ValueLayout.ADDRESS));
|
||||
|
||||
private static final int MAX_FRAME = 5760;
|
||||
|
||||
private final Arena arena = Arena.ofConfined();
|
||||
private final MemorySegment decoder;
|
||||
private final MemorySegment pcm;
|
||||
private final MemorySegment data = arena.allocate(1 << 20);
|
||||
final int channels;
|
||||
|
||||
LibOpusMultistream(int channels, int streams, int coupled, byte[] mapping) throws Throwable {
|
||||
this.channels = channels;
|
||||
MemorySegment map = arena.allocate(mapping.length);
|
||||
map.copyFrom(MemorySegment.ofArray(mapping));
|
||||
MemorySegment error = arena.allocate(ValueLayout.JAVA_INT);
|
||||
decoder = (MemorySegment) CREATE.invokeExact(48_000, channels, streams, coupled, map, error);
|
||||
if (error.get(ValueLayout.JAVA_INT, 0) != 0) throw new IllegalStateException("create: " + error.get(ValueLayout.JAVA_INT, 0));
|
||||
pcm = arena.allocate(ValueLayout.JAVA_FLOAT, (long) MAX_FRAME * channels);
|
||||
}
|
||||
|
||||
/** @return interleaved samples, or throws if libopus rejects the packet */
|
||||
float[] decode(byte[] packet) throws Throwable {
|
||||
MemorySegment.copy(packet, 0, data, ValueLayout.JAVA_BYTE, 0, packet.length);
|
||||
int n = (int) DECODE.invokeExact(decoder, data, packet.length, pcm, MAX_FRAME, 0);
|
||||
if (n < 0) throw new IllegalStateException("opus_multistream_decode_float: " + n);
|
||||
return pcm.asSlice(0, (long) n * channels * 4).toArray(ValueLayout.JAVA_FLOAT);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
try {
|
||||
DESTROY.invokeExact(decoder);
|
||||
} catch (Throwable ignored) {
|
||||
}
|
||||
arena.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
package com.ts3client.recording;
|
||||
|
||||
import com.ts3client.audio.VoiceFormat;
|
||||
import com.ts3client.audio.desktop.NativeOpusCodec;
|
||||
import com.ts3client.audio.ogg.SelfDelimitedOpus;
|
||||
import com.ts3client.audio.opus.OpusEncoder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Recordings decode, through libopus' own multistream decoder, to every speaker where and
|
||||
* when they spoke.
|
||||
*/
|
||||
class VoiceRecorderTest {
|
||||
|
||||
private static final int FRAME = VoiceFormat.FRAME_SIZE;
|
||||
private static final long FRAME_NANOS = TimeUnit.MILLISECONDS.toNanos(20);
|
||||
/** Frames the simulated recordings run for. */
|
||||
private static final int LENGTH = 250;
|
||||
|
||||
@TempDir
|
||||
Path dir;
|
||||
|
||||
private final NativeOpusCodec opus = new NativeOpusCodec();
|
||||
|
||||
/** Who speaks when, in frames, at which pitch. */
|
||||
private record Burst(String uid, String nickname, int from, int to, double hz) {
|
||||
}
|
||||
|
||||
private static final List<Burst> CONVERSATION = List.of(
|
||||
new Burst("uidA", "Alice (renamed)", 50, 100, 440),
|
||||
new Burst("uidB", "Bob", 150, 175, 660),
|
||||
// After a pause long enough for a fresh encoder.
|
||||
new Burst("uidA", "Alice (renamed)", 200, 210, 440));
|
||||
|
||||
@Test
|
||||
void multitrackPutsEverySpeakerOnTheirOwnChannelFromTheStart() throws Throwable {
|
||||
Path file = dir.resolve("multi.ogg");
|
||||
List<Path> files = record(VoiceRecorder.Mode.MULTITRACK, file, Map.of("uidA", "Alice"), CONVERSATION);
|
||||
assertEquals(List.of(file), files);
|
||||
|
||||
Decoded d = decode(file);
|
||||
assertEquals(2, d.channels);
|
||||
assertTrue(d.comments.containsAll(List.of("CHANNEL1_NAME=Alice", "CHANNEL1_UID=uidA",
|
||||
"CHANNEL2_NAME=Bob", "CHANNEL2_UID=uidB")), d.comments.toString());
|
||||
assertEquals(LENGTH * FRAME, d.frames());
|
||||
|
||||
assertTone(d, 0, 50, 100, 440);
|
||||
assertSilent(d, 0, 0, 49);
|
||||
assertSilent(d, 0, 102, 199);
|
||||
assertTone(d, 0, 200, 210, 440);
|
||||
assertTone(d, 1, 150, 175, 660);
|
||||
assertStartsAt(d, 0, 50);
|
||||
assertStartsAt(d, 0, 200);
|
||||
assertStartsAt(d, 1, 150);
|
||||
assertSilent(d, 1, 0, 149);
|
||||
assertSilent(d, 1, 177, LENGTH);
|
||||
assertTrue(Files.list(dir).noneMatch(p -> p.getFileName().toString().startsWith(".")), "spool left behind");
|
||||
}
|
||||
|
||||
@Test
|
||||
void mixedPutsEveryoneOnOneMonoTrack() throws Throwable {
|
||||
Path file = dir.resolve("mixed.ogg");
|
||||
record(VoiceRecorder.Mode.MIXED, file, Map.of(), CONVERSATION);
|
||||
|
||||
Decoded d = decode(file);
|
||||
assertEquals(1, d.channels);
|
||||
assertEquals(LENGTH * FRAME, d.frames());
|
||||
assertTone(d, 0, 50, 100, 440);
|
||||
assertTone(d, 0, 150, 175, 660);
|
||||
assertSilent(d, 0, 0, 49);
|
||||
assertSilent(d, 0, 102, 149);
|
||||
}
|
||||
|
||||
@Test
|
||||
void moreSpeakersThanAFileHoldsContinueInTheNextFile() throws Throwable {
|
||||
List<Burst> crowd = new ArrayList<>();
|
||||
for (int i = 0; i < 300; i++) crowd.add(new Burst("uid" + i, "Client " + i, i % 200, i % 200 + 5, 440));
|
||||
crowd.sort((a, b) -> Integer.compare(a.from, b.from));
|
||||
Path file = dir.resolve("crowd.ogg");
|
||||
List<Path> files = record(VoiceRecorder.Mode.MULTITRACK, file, Map.of(), crowd);
|
||||
assertEquals(List.of(file, dir.resolve("crowd-2.ogg")), files);
|
||||
|
||||
Decoded first = decode(files.get(0));
|
||||
Decoded second = decode(files.get(1));
|
||||
assertEquals(255, first.channels);
|
||||
assertEquals(45, second.channels);
|
||||
assertEquals(LENGTH * FRAME, first.frames());
|
||||
assertEquals(LENGTH * FRAME, second.frames());
|
||||
// Channels go in the order speakers first spoke.
|
||||
assertTone(first, 0, 0, 5, 440);
|
||||
// Two speakers a frame start until frame 100, one after: the 256th first spoke at 155.
|
||||
assertTone(second, 0, 155, 160, 440);
|
||||
assertStartsAt(second, 0, 155);
|
||||
assertSilent(second, 0, 0, 154);
|
||||
}
|
||||
|
||||
@Test
|
||||
void multiFramePacketsAreSplitWhereLibopusSplitsThem() throws Throwable {
|
||||
byte[][] f = new byte[6][];
|
||||
try (OpusEncoder encoder = RecordingEncoders.mono(opus, 48_000)) {
|
||||
for (int i = 0; i < f.length; i++) f[i] = encoder.encode(tone(440 + 50 * i, i));
|
||||
}
|
||||
byte[] padded = code3(f[0], f[1], 3);
|
||||
byte[] code2 = cat(new byte[]{(byte) ((f[2][0] & 0xfc) | 2), (byte) (f[2].length - 1)},
|
||||
Arrays.copyOfRange(f[2], 1, f[2].length), Arrays.copyOfRange(f[3], 1, f[3].length));
|
||||
byte[] last = code3(f[4], f[5], 0);
|
||||
byte[] joined = SelfDelimitedOpus.join(List.of(padded, code2, last));
|
||||
|
||||
try (LibOpusMultistream decoder = new LibOpusMultistream(3, 3, 0, new byte[]{0, 1, 2})) {
|
||||
assertEquals(3 * 2 * FRAME, decoder.decode(joined).length);
|
||||
}
|
||||
}
|
||||
|
||||
/** Two single-frame packets of one configuration as one VBR code 3 packet. */
|
||||
private static byte[] code3(byte[] a, byte[] b, int padding) {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
out.write((a[0] & 0xfc) | 3);
|
||||
out.write(0x80 | (padding > 0 ? 0x40 : 0) | 2);
|
||||
if (padding > 0) out.write(padding);
|
||||
out.write(a.length - 1);
|
||||
out.write(a, 1, a.length - 1);
|
||||
out.write(b, 1, b.length - 1);
|
||||
out.write(new byte[padding], 0, padding);
|
||||
return out.toByteArray();
|
||||
}
|
||||
|
||||
private static byte[] cat(byte[]... parts) {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
for (byte[] p : parts) out.writeBytes(p);
|
||||
return out.toByteArray();
|
||||
}
|
||||
|
||||
// ---- simulation ----
|
||||
|
||||
private List<Path> record(VoiceRecorder.Mode mode, Path file, Map<String, String> names, List<Burst> bursts)
|
||||
throws Exception {
|
||||
AtomicLong clock = new AtomicLong(1_000_000_000L);
|
||||
long start = clock.get();
|
||||
VoiceRecorder recorder = VoiceRecorder.start(mode, file, opus, names, clock::get);
|
||||
for (int f = 0; f < LENGTH; f++) {
|
||||
for (Burst burst : bursts) {
|
||||
if (f < burst.from || f >= burst.to) continue;
|
||||
// Played a little late, as audio threads are.
|
||||
clock.set(start + f * FRAME_NANOS + (f * 7919L % 5) * 1_000_000L);
|
||||
recorder.offer(burst.uid, burst.nickname, tone(burst.hz, f), 1, FRAME);
|
||||
}
|
||||
}
|
||||
clock.set(start + LENGTH * FRAME_NANOS);
|
||||
List<Path> files = recorder.stop().get(60, TimeUnit.SECONDS);
|
||||
String dump = System.getProperty("recording.dump");
|
||||
if (dump != null) {
|
||||
for (Path p : files) Files.copy(p, Path.of(dump).resolve(p.getFileName()), StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
private static float[] tone(double hz, int frame) {
|
||||
float[] pcm = new float[FRAME];
|
||||
for (int i = 0; i < FRAME; i++) {
|
||||
pcm[i] = (float) (0.5 * Math.sin(2 * Math.PI * hz * ((long) frame * FRAME + i) / VoiceFormat.SAMPLE_RATE));
|
||||
}
|
||||
return pcm;
|
||||
}
|
||||
|
||||
// ---- checks ----
|
||||
|
||||
/** A tone of {@code hz} over the frames, give or take the edges. */
|
||||
private static void assertTone(Decoded d, int channel, int from, int to, double hz) {
|
||||
int start = (from + 1) * FRAME;
|
||||
int end = (to - 1) * FRAME;
|
||||
double energy = 0;
|
||||
double sin = 0;
|
||||
double cos = 0;
|
||||
for (int i = start; i < end; i++) {
|
||||
double x = d.sample(i, channel);
|
||||
double phase = 2 * Math.PI * hz * i / VoiceFormat.SAMPLE_RATE;
|
||||
sin += x * Math.sin(phase);
|
||||
cos += x * Math.cos(phase);
|
||||
energy += x * x;
|
||||
}
|
||||
double amplitude = Math.sqrt(sin * sin + cos * cos) * 2 / (end - start);
|
||||
double rms = Math.sqrt(energy / (end - start));
|
||||
assertTrue(amplitude > 0.4, "channel " + channel + " frames " + from + "-" + to + ": amplitude at " + hz + " Hz " + amplitude);
|
||||
assertTrue(rms > 0.3 && rms < 0.42, "channel " + channel + " frames " + from + "-" + to + ": rms " + rms);
|
||||
}
|
||||
|
||||
/** Sound sets in right at the frame, not faded in. */
|
||||
private static void assertStartsAt(Decoded d, int channel, int frame) {
|
||||
int onset = frame * FRAME - FRAME;
|
||||
while (Math.abs(d.sample(onset, channel)) < 0.05) onset++;
|
||||
assertEquals(frame * FRAME, onset, 48.0, "channel " + channel + " onset");
|
||||
}
|
||||
|
||||
private static void assertSilent(Decoded d, int channel, int from, int to) {
|
||||
double peak = 0;
|
||||
for (int i = from * FRAME; i < Math.min(to * FRAME, d.frames()); i++) peak = Math.max(peak, Math.abs(d.sample(i, channel)));
|
||||
assertTrue(peak < 0.01, "channel " + channel + " frames " + from + "-" + to + ": peak " + peak);
|
||||
}
|
||||
|
||||
// ---- decoding ----
|
||||
|
||||
private record Decoded(int channels, List<String> comments, float[] pcm) {
|
||||
int frames() {
|
||||
return pcm.length / channels;
|
||||
}
|
||||
|
||||
double sample(int i, int channel) {
|
||||
return pcm[i * channels + channel];
|
||||
}
|
||||
}
|
||||
|
||||
private static Decoded decode(Path file) throws Throwable {
|
||||
List<byte[]> packets = new ArrayList<>();
|
||||
long granule = readOgg(Files.readAllBytes(file), packets);
|
||||
|
||||
ByteBuffer head = ByteBuffer.wrap(packets.get(0)).order(ByteOrder.LITTLE_ENDIAN);
|
||||
assertEquals("OpusHead", new String(packets.get(0), 0, 8, StandardCharsets.US_ASCII));
|
||||
int channels = head.get(9) & 0xff;
|
||||
int preSkip = head.getShort(10) & 0xffff;
|
||||
int family = head.get(18) & 0xff;
|
||||
int streams = 1;
|
||||
int coupled = 0;
|
||||
byte[] mapping = {0};
|
||||
if (family != 0) {
|
||||
assertEquals(255, family);
|
||||
streams = head.get(19) & 0xff;
|
||||
coupled = head.get(20) & 0xff;
|
||||
mapping = Arrays.copyOfRange(packets.get(0), 21, 21 + channels);
|
||||
}
|
||||
|
||||
ByteBuffer tags = ByteBuffer.wrap(packets.get(1)).order(ByteOrder.LITTLE_ENDIAN);
|
||||
tags.position(8);
|
||||
int vendor = tags.getInt();
|
||||
tags.position(tags.position() + vendor);
|
||||
int count = tags.getInt();
|
||||
List<String> comments = new ArrayList<>();
|
||||
for (int i = 0; i < count; i++) {
|
||||
byte[] s = new byte[tags.getInt()];
|
||||
tags.get(s);
|
||||
comments.add(new String(s, StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
float[] pcm = new float[0];
|
||||
int length = 0;
|
||||
try (LibOpusMultistream decoder = new LibOpusMultistream(channels, streams, coupled, mapping)) {
|
||||
for (byte[] packet : packets.subList(2, packets.size())) {
|
||||
float[] out = decoder.decode(packet);
|
||||
if (length + out.length > pcm.length) pcm = Arrays.copyOf(pcm, Math.max(pcm.length * 2, length + out.length));
|
||||
System.arraycopy(out, 0, pcm, length, out.length);
|
||||
length += out.length;
|
||||
}
|
||||
}
|
||||
assertTrue(granule * channels <= length, "granule past the audio");
|
||||
return new Decoded(channels, comments, Arrays.copyOfRange(pcm, preSkip * channels, (int) granule * channels));
|
||||
}
|
||||
|
||||
/** Reads every packet, checking the pages' framing; returns the last page's granule position. */
|
||||
private static long readOgg(byte[] file, List<byte[]> packets) {
|
||||
ByteBuffer in = ByteBuffer.wrap(file).order(ByteOrder.LITTLE_ENDIAN);
|
||||
ByteArrayOutputStream packet = new ByteArrayOutputStream();
|
||||
long granule = -1;
|
||||
int expectedSequence = 0;
|
||||
boolean sawLast = false;
|
||||
while (in.hasRemaining()) {
|
||||
int start = in.position();
|
||||
assertEquals(0x5367674f, in.getInt(), "page sync");
|
||||
in.get();
|
||||
int flags = in.get() & 0xff;
|
||||
granule = in.getLong();
|
||||
in.getInt();
|
||||
assertEquals(expectedSequence++, in.getInt(), "page sequence");
|
||||
in.getInt();
|
||||
assertEquals(packet.size() > 0, (flags & 1) != 0, "continuation flag");
|
||||
assertEquals(start == 0, (flags & 2) != 0, "first page flag");
|
||||
sawLast = (flags & 4) != 0;
|
||||
int segments = in.get() & 0xff;
|
||||
byte[] lacing = new byte[segments];
|
||||
in.get(lacing);
|
||||
for (byte l : lacing) {
|
||||
int size = l & 0xff;
|
||||
byte[] data = new byte[size];
|
||||
in.get(data);
|
||||
packet.writeBytes(data);
|
||||
if (size < 255) {
|
||||
packets.add(packet.toByteArray());
|
||||
packet.reset();
|
||||
}
|
||||
}
|
||||
}
|
||||
assertTrue(sawLast, "last page flag");
|
||||
return granule;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user