"""Rendering: impulse responses applied to dry signals (auralisation).
- :func:`convolve`: offline convolution of a recording with an impulse response (FFT
overlap-add in float64), for any number of response channels.
- :func:`resample`: band-limited sample-rate conversion, so a 44.1 kHz recording can go
through a 48 kHz response.
- :func:`crossfaded`: a crossfade from one sound to another whose power runs on however
alike the two are, broadband or band by band: a tour's cuts from one seat to the
next, a walk's from one place to the next.
- :func:`auralize`: the sources of a simulation each playing their own signal, heard at
one listener in a spatial format (mono, binaural, Ambisonics, loudspeakers), summed.
Outdoors in turbulence the early paths change as the eddies drift past: a new
moment of them every :data:`MOMENTS_PER_CORRELATION`-th of their correlation time (at
most every :data:`MOMENT_MIN_S`), each path's delay going from one moment's to the
next's, so the sound flickers as it does in gusty air.
- :class:`Convolver`: the compiled real-time convolver, block by block. It divides the
response into partitions that grow along it, from the block size up to 8192 frames,
and multiplies in the frequency domain, so the only latency is the block itself;
nothing allocates while it runs.
- :func:`play`: a signal, or live input, through a response on the sound card (needs the
``measure`` extra: sounddevice); or through any block processor with one input, such
as :class:`~acoustic_spaces.headtracking.HeadTrackedBinaural`.
"""
from __future__ import annotations
import dataclasses
import inspect
import math
import threading
import time
from collections.abc import Mapping
from dataclasses import dataclass
from fractions import Fraction
from typing import TYPE_CHECKING
import numpy as np
from numpy.lib.stride_tricks import sliding_window_view
from acoustic_spaces import products
if TYPE_CHECKING:
from acoustic_spaces.engine.simulate import SimulationResult
from acoustic_spaces.headtracking import HeadTrackedBinaural
from acoustic_spaces.spatial import Format
#: Smallest FFT of the offline overlap-add.
_MIN_FFT = 1 << 14
#: Zero crossings of the resampling filter on each side, and its Kaiser window's beta.
_RESAMPLE_ZEROS = 32
_RESAMPLE_BETA = 9.0
#: Most filter phases resampling keeps; finer ratios round the position to these.
_MAX_PHASES = 4096
#: Output samples resampled at once when the rates have no short ratio.
_RESAMPLE_CHUNK = 1 << 15
#: In turbulence, the moments of the eddies a signal goes through: this many per
#: correlation time, and at most one every MOMENT_MIN_S seconds.
MOMENTS_PER_CORRELATION = 16
MOMENT_MIN_S = 0.05
def _pow2(n: int) -> int:
return 1 << max(0, int(n) - 1).bit_length()
def _as_2d(x: np.ndarray, name: str) -> np.ndarray:
x = np.asarray(x, dtype=np.float64)
if x.ndim == 1:
x = x[:, None]
if x.ndim != 2 or x.shape[0] == 0 or x.shape[1] == 0:
raise ValueError(f"{name} must be a non-empty (samples,) or (samples, channels) array")
return x
[docs]
def convolve(signal: np.ndarray, ir: np.ndarray) -> np.ndarray:
"""Full linear convolution: len(signal) + len(ir) - 1 samples.
``signal`` and ``ir`` are (samples,) or (samples, channels). A mono signal goes
through every channel of the response (a dry source through a binaural or Ambisonics
response); a multichannel signal goes through a mono response channel by channel.
The result is 1-D only when both inputs are.
"""
x, h = _as_2d(signal, "signal"), _as_2d(ir, "ir")
if x.shape[1] > 1 and h.shape[1] > 1:
raise ValueError(
"a multichannel signal needs a mono response; mix the signal down, or use "
"auralize for several sources"
)
n, m = len(x), len(h)
length = n + m - 1
nfft = min(_pow2(length), max(_pow2(4 * m), _MIN_FFT))
step = nfft - m + 1
spectrum = np.fft.rfft(h, nfft, axis=0)
out = np.zeros((length, max(x.shape[1], h.shape[1])))
for start in range(0, n, step):
chunk = np.fft.rfft(x[start : start + step], nfft, axis=0)
y = np.fft.irfft(chunk * spectrum, nfft, axis=0)
stop = min(start + nfft, length)
out[start:stop] += y[: stop - start]
if np.ndim(signal) == 1 and np.ndim(ir) == 1:
return out[:, 0]
return out
def _kaiser_sinc(t: np.ndarray, cutoff: float) -> np.ndarray:
"""Low-pass taps at offsets ``t`` (input samples), ``cutoff`` as a share of Nyquist."""
half = _RESAMPLE_ZEROS / cutoff
window = np.i0(_RESAMPLE_BETA * np.sqrt(np.clip(1.0 - (t / half) ** 2, 0.0, 1.0)))
return cutoff * np.sinc(cutoff * t) * window / np.i0(_RESAMPLE_BETA)
[docs]
def resample(signal: np.ndarray, rate_in: float, rate_out: float) -> np.ndarray:
"""``signal`` (samples,) or (samples, channels) at ``rate_out``.
Polyphase windowed-sinc interpolation (Kaiser, 32 zero crossings each side), with
the cutoff just below the lower Nyquist frequency so nothing aliases.
"""
x = np.asarray(signal, dtype=np.float64)
if rate_in <= 0 or rate_out <= 0:
raise ValueError("sample rates must be positive")
if math.isclose(rate_in, rate_out) or len(x) == 0:
return x.copy()
# Output sample k sits at input position k * step / phases.
ratio = (Fraction(rate_in) / Fraction(rate_out)).limit_denominator(1 << 20)
exact = ratio.denominator <= _MAX_PHASES
step, phases = (ratio.numerator, ratio.denominator) if exact else (0, _MAX_PHASES)
cutoff = 0.97 * min(1.0, rate_out / rate_in)
reach = int(math.ceil(_RESAMPLE_ZEROS / cutoff))
offsets = np.arange(-reach + 1, reach + 1) # taps relative to floor(position)
frac = np.arange(phases) / phases
table = _kaiser_sinc(offsets[None, :] - frac[:, None], cutoff) # (phases, taps)
two_d = x.reshape(len(x), -1)
count = int(math.ceil(len(x) * rate_out / rate_in))
out = np.zeros((count, two_d.shape[1]))
for c in range(two_d.shape[1]):
padded = np.concatenate([np.zeros(reach), two_d[:, c], np.zeros(reach + 1)])
# Row r holds the taps around input sample r - 1.
rows = sliding_window_view(padded, len(offsets))
if exact:
# Outputs k0, k0 + phases, ... share phase p and step `step` samples apart, so
# each phase is one matrix-vector product.
inverse = pow(step, -1, phases)
for p in range(phases):
k0 = p * inverse % phases
if k0 >= count:
continue
n = len(range(k0, count, phases))
first = (k0 * step - p) // phases + 1
out[k0::phases, c] = products.product(
rows[first : first + (n - 1) * step + 1 : step], table[p]
)
continue
for start in range(0, count, _RESAMPLE_CHUNK):
k = np.arange(start, min(start + _RESAMPLE_CHUNK, count))
position = k * (rate_in / rate_out)
whole = np.floor(position).astype(np.int64)
phase = np.round((position - whole) * phases).astype(np.int64)
whole, phase = whole + phase // phases, phase % phases
out[k, c] = np.einsum("kt,kt->k", table[phase], rows[whole + 1])
return out[:, 0] if x.ndim == 1 else out
[docs]
def power_gains(sources, names) -> dict[str, float]:
"""Amplitude gain of each source in ``names`` for its sound power: 1 for the loudest
of them, 10^(-ΔL/20) for one ΔL dB below it. ``sources`` are scene sources, or a
mapping of name to sound power level (dB re 1 pW); names it lacks get no gain."""
if not isinstance(sources, Mapping):
sources = {source.name: source.sound_power_db for source in sources}
levels = {name: float(sources[name]) for name in names if name in sources}
if not levels:
return {}
loudest = max(levels.values())
return {name: 10.0 ** ((level - loudest) / 20.0) for name, level in levels.items()}
[docs]
def crossfaded(
x0: np.ndarray, x1: np.ndarray, ramp: np.ndarray, sample_rate: float | None = None
) -> np.ndarray:
"""``x0`` fading out and ``x1`` fading in, (n,) or (n, channels): ``x1``'s gain is
``ramp`` (n, rising from 0 to 1) and ``x0``'s sqrt(1 - ramp²), equal power for two
unlike sounds, both scaled, sample by sample, by how alike the two are over the
crossfade (their correlation coefficient, one for all channels), so that the power
runs from one's to the other's as it would for unlike ones. Alike sounds would
otherwise be up to 3 dB louder in the middle, sounds in antiphase silent. The
scaling is at most +6 dB: where the two cancel to less than half the amplitude
unlike sounds would have (near antiphase, in the middle of the crossfade), their
sum stays below it. With ``sample_rate``, each octave band
(:func:`~acoustic_spaces.synthesis.split_bands`) by its own correlation."""
x0, x1 = np.asarray(x0, dtype=float), np.asarray(x1, dtype=float)
ramp = np.asarray(ramp, dtype=float)
if sample_rate is not None:
from acoustic_spaces.synthesis import split_bands
bands = zip(split_bands(x0, sample_rate), split_bands(x1, sample_rate), strict=True)
return sum((crossfaded(b0, b1, ramp) for b0, b1 in bands), np.zeros_like(x0))
fade_in, fade_out = ramp, np.sqrt(np.clip(1.0 - ramp**2, 0.0, None))
shape = (-1,) + (1,) * (x0.ndim - 1)
both = x0 * fade_out.reshape(shape) + x1 * fade_in.reshape(shape)
p0, p1 = float(np.mean(x0**2)), float(np.mean(x1**2))
if p0 <= 0.0 or p1 <= 0.0:
return both
rho = float(np.mean(x0 * x1)) / math.sqrt(p0 * p1)
unlike = fade_out**2 * p0 + fade_in**2 * p1
alike = unlike + 2.0 * rho * fade_out * fade_in * math.sqrt(p0 * p1)
return both * np.sqrt(unlike / np.maximum(alike, unlike / 4.0)).reshape(shape)
[docs]
def auralize(
result: SimulationResult,
signals: Mapping[str, np.ndarray | tuple[np.ndarray, float]],
listener: str | None = None,
fmt: Format | None = None,
*,
sample_rate: int | None = None,
seed: int | None = None,
equal_power: bool = False,
) -> np.ndarray:
"""Every source in ``signals`` playing its signal, heard at ``listener``.
``signals`` maps source names to a signal at ``sample_rate``, or to a
(signal, rate) pair, resampled when the rates differ; multichannel signals are mixed
down, as sources are points. ``fmt`` is a format of :mod:`acoustic_spaces.spatial`
(default mono, through the listener's microphone when it has a pattern:
:func:`~acoustic_spaces.spatial.for_listener`); ``listener`` may be left out when
the result has one listener.
Returns (samples, channels) at ``sample_rate`` (default: the simulation's), with the
sources at the levels the simulation gives them: the responses carry each source's
feed (gain, equalizer, delay) but are per unit of sound power, so each signal is
scaled by its source's sound power (``level_db``) against the loudest source's in
``signals`` (:func:`power_gains`). ``equal_power`` leaves that out, as if every
source had the loudest one's power.
"""
from acoustic_spaces.spatial import Mono, for_listener
sample_rate = int(sample_rate or result.settings.sample_rate)
listeners = sorted({lis for _, lis in result.responses})
if listener is None:
if len(listeners) != 1:
raise ValueError(f"pick a listener: {', '.join(listeners)}")
listener = listeners[0]
items = {item.name: item for item in result.room.scene.listeners}
fmt = for_listener(fmt or Mono(), items.get(listener))
if not signals:
raise ValueError("no signals to auralize")
mix = np.zeros((0, fmt.channels))
gains = {} if equal_power else power_gains(result.room.scene.sources, signals)
for source, value in signals.items():
if (source, listener) not in result.responses:
sources = sorted({s for s, lis in result.responses if lis == listener})
raise ValueError(f"no response from {source!r} at {listener!r}; sources: {sources}")
signal, rate = value if isinstance(value, tuple) else (value, sample_rate)
dry = _as_2d(signal, f"signal of {source!r}").mean(axis=1) * gains.get(source, 1.0)
dry = resample(dry, rate, sample_rate)
response = result[source, listener]
if flickers(response):
n = int(math.ceil(response.length_s * sample_rate))
gusts = gusts_for(response, (len(dry) + n) / sample_rate, seed)
wet = through_eddies(dry, response, fmt, sample_rate, seed, gusts, 0, len(dry) + n - 1)
else:
ir = response.spatial_impulse_response(fmt, sample_rate, seed)
wet = convolve(dry, ir)
if len(wet) > len(mix):
mix = np.concatenate([mix, np.zeros((len(wet) - len(mix), fmt.channels))])
mix[: len(wet)] += wet
return mix
def flickers(response) -> bool:
"""Whether ``response``'s early paths change as eddies drift past (outdoors in
turbulence, without the wave solver's bands). Anything without early paths does
not."""
early = getattr(response, "early", None)
return getattr(early, "wander", None) is not None and getattr(response, "wave", None) is None
def gusts_for(response, duration_s: float, seed: int | None = None):
"""The eddies' moments (:class:`~acoustic_spaces.engine.turbulence.Gusts`) for
``duration_s`` seconds of sound through ``response``'s turbulence, drawn with
``seed`` (default the response's)."""
from acoustic_spaces.engine.turbulence import Gusts
correlation = response.early.wander.correlation_s
step = max(correlation / MOMENTS_PER_CORRELATION, MOMENT_MIN_S)
count = int(math.ceil(duration_s / step)) + 2
return Gusts(correlation, step, count, response.seed if seed is None else seed)
def through_eddies(
dry: np.ndarray, response, fmt, sample_rate: int, seed, gusts, begin: int, end: int
) -> np.ndarray:
"""(end - begin, channels) samples ``begin`` to ``end`` of the mono ``dry`` signal
played through ``response`` in ``fmt``: its early paths each through its own delay,
which the eddies ``gusts`` has move from one moment to the next (taken between the
moments on either side, as :func:`acoustic_spaces.doppler.render` takes them with
the one place), its late part as it is. Crossfading the moments' impulse responses
instead would mix each path at two delays a few microseconds apart, which cancels
the treble half-way through each crossfade."""
from acoustic_spaces import doppler
from acoustic_spaces.engine.early import EarlyReflections
out = np.zeros((end - begin, fmt.channels))
quiet = dataclasses.replace(response, early=EarlyReflections.empty(response.late.shape[0]))
_add_window(out, dry, quiet.spatial_impulse_response(fmt, sample_rate, seed), begin, end, None)
out += doppler.render(
dry,
[response],
np.zeros(1),
fmt,
sample_rate,
end,
source_walks=False,
planes=np.zeros(0, dtype=np.int64),
step_s=0.0,
gusts=gusts,
seed=seed,
span=(begin, end),
)
return out
def _add_window(out, dry, ir, lo: int, hi: int, weight, origin: int | None = None) -> None:
"""Adds samples ``lo`` to ``hi`` of ``dry`` convolved with ``ir``, times ``weight``
(or 1), to ``out``, whose first sample is sample ``origin`` (default ``lo``)."""
origin = lo if origin is None else origin
first = max(lo - len(ir) + 1, 0)
piece = dry[first:hi]
if len(piece) == 0:
return
wet = convolve(piece, ir)[lo - first : hi - first]
if wet.ndim == 1:
wet = wet[:, None]
if weight is not None:
wet = wet * weight[: len(wet), None]
out[lo - origin : lo - origin + len(wet)] += wet
[docs]
class Convolver:
"""Real-time convolution through ``ir``, one block of ``block`` frames at a time.
``ir`` is (taps,), (taps, outputs) for one input, or (taps, inputs, outputs), where
each output sums its inputs, each through its own response. ``block`` is a power of
two, and the only latency. The response is split into segments whose partitions
grow along it (see :attr:`segments`), and the work of the long partitions is spread
over the blocks in between, so the cost per block stays even and grows slowly with
the response's length: a 3 s, 16-channel response in blocks of 128 frames takes
about 0.3 ms of the 2.7 ms each block lasts at 48 kHz, on one current x86-64 core.
"""
def __init__(self, ir: np.ndarray, block: int = 256) -> None:
from acoustic_spaces import _native
h = np.asarray(ir, dtype=np.float32)
if h.ndim == 1:
h = h[:, None, None]
elif h.ndim == 2:
h = h[:, None, :]
if h.ndim != 3 or 0 in h.shape:
raise ValueError("ir must be (taps,), (taps, outputs) or (taps, inputs, outputs)")
if block < 2 or block & (block - 1):
raise ValueError("block must be a power of two")
self._native = _native.Convolver(np.ascontiguousarray(h.transpose(1, 2, 0)), int(block))
@property
def inputs(self) -> int:
return self._native.inputs
@property
def outputs(self) -> int:
return self._native.outputs
@property
def taps(self) -> int:
return self._native.taps
@property
def block(self) -> int:
return self._native.block
@property
def segments(self) -> list[tuple[int, int, int]]:
"""(first tap, partition length, partitions) of each segment of the response."""
return list(self._native.segments)
[docs]
def process(self, block: np.ndarray, out: np.ndarray | None = None) -> np.ndarray:
"""One block, (block,) or (block, inputs), to (block, outputs) float32.
Given ``out``, a writable C-contiguous float32 (block, outputs) array, the block
is written into it and nothing new is allocated for it, as a sound card's
callback needs (a float32 C-contiguous ``block`` is not copied either).
"""
x = np.asarray(block, dtype=np.float32)
if x.ndim == 1:
x = x[:, None]
x = np.ascontiguousarray(x)
if out is None:
return self._native.process(x)
if out.dtype != np.float32 or not out.flags.c_contiguous or not out.flags.writeable:
raise ValueError("out must be a writable C-contiguous float32 array")
self._native.process_into(x, out)
return out
[docs]
def run(self, signal: np.ndarray) -> np.ndarray:
"""A whole signal, (samples,) or (samples, inputs), through the convolver:
(samples + taps - 1, outputs), as :func:`convolve` gives it."""
x = np.asarray(signal, dtype=np.float32)
x = x[:, None] if x.ndim == 1 else x
if x.ndim != 2 or x.shape[1] != self.inputs:
raise ValueError(f"signal must have {self.inputs} channel(s)")
length = len(x) + self.taps - 1
blocks = -(-length // self.block)
padded = np.zeros((blocks * self.block, self.inputs), np.float32)
padded[: len(x)] = x
out = np.concatenate(
[self.process(padded[k * self.block : (k + 1) * self.block]) for k in range(blocks)]
)
return out[:length]
[docs]
def reset(self) -> None:
"""Forget all past input."""
self._native.reset()
[docs]
@dataclass
class Playback:
"""How a :func:`play` run went."""
frames: int
#: Blocks the sound card reported an underflow or overflow for.
xruns: int
#: Longest time spent convolving one block, as a share of the block's duration.
load: float
[docs]
def play(
ir: np.ndarray | Convolver | HeadTrackedBinaural,
sample_rate: int,
signal: np.ndarray | None = None,
*,
block: int = 256,
gain: float = 1.0,
device: object = None,
seconds: float | None = None,
stop: threading.Event | None = None,
) -> Playback:
"""Play ``signal`` through ``ir`` (samples, outputs) on the sound card, convolving in
real time; without a signal, the sound card's first input goes through it live.
``ir`` may also be a processor that works block by block, as a :class:`Convolver`
or :class:`~acoustic_spaces.headtracking.HeadTrackedBinaural` does; its own block
size then replaces ``block``. Playback ends with the signal's reverberation tail;
live input runs for ``seconds``, until ``stop`` is set, or until Ctrl-C. ``signal``
must be at ``sample_rate``.
"""
from acoustic_spaces.measurement import _sounddevice
sd = _sounddevice()
convolver = ir if hasattr(ir, "process") else Convolver(ir, block)
block = convolver.block
if convolver.inputs != 1:
raise ValueError("play takes a response with one input")
dry = None if signal is None else _as_2d(signal, "signal").mean(axis=1).astype(np.float32)
total = None if dry is None else len(dry) + convolver.taps - 1
if dry is None and seconds is not None:
total = int(round(seconds * sample_rate))
budget = block / sample_rate
state = {"frames": 0, "xruns": 0, "load": 0.0}
done = threading.Event()
scratch = np.zeros(block, np.float32)
# A processor that takes ``out`` writes each block into this one buffer, so the
# sound card's callback allocates nothing for it.
wet = np.zeros((block, convolver.outputs), np.float32)
into = "out" in inspect.signature(convolver.process).parameters
def render(frames_in: np.ndarray, outdata: np.ndarray, status) -> None:
began = time.perf_counter()
if status:
state["xruns"] += 1
processed = convolver.process(frames_in, out=wet) if into else convolver.process(frames_in)
np.multiply(processed, gain, out=outdata)
state["load"] = max(state["load"], (time.perf_counter() - began) / budget)
state["frames"] += block
if (total is not None and state["frames"] >= total) or (stop and stop.is_set()):
raise sd.CallbackStop
def output_callback(outdata, frames, _time, status) -> None:
start = state["frames"]
scratch[:] = 0.0
if start < len(dry):
piece = dry[start : start + frames]
scratch[: len(piece)] = piece
render(scratch, outdata, status)
def duplex_callback(indata, outdata, frames, _time, status) -> None:
render(indata[:, 0], outdata, status)
common = {
"samplerate": sample_rate,
"blocksize": block,
"dtype": "float32",
"device": device,
"finished_callback": done.set,
}
if dry is not None:
stream = sd.OutputStream(channels=convolver.outputs, callback=output_callback, **common)
else:
stream = sd.Stream(channels=(1, convolver.outputs), callback=duplex_callback, **common)
with stream:
try:
while not done.wait(0.1): # a timeout keeps Ctrl-C working on Windows
pass
except KeyboardInterrupt:
pass
return Playback(state["frames"], state["xruns"], state["load"])
__all__ = [
"Convolver",
"Playback",
"auralize",
"convolve",
"crossfaded",
"play",
"power_gains",
"resample",
]