mirror of
https://github.com/blender/blender
synced 2026-09-29 04:37:17 +03:00
Geometry Nodes: new Sample Sound Frequencies node
This adds a new node which allows getting frequency information from sound directly inside of Geometry Nodes, making baking to f-curves unnecessary. This is a new implementation, but it is significantly inspired by #122228 and also builds on top of some of the changes that landed during the GSOC project already (like `PROP_FREQUENCY`). This also exposes the new sound socket types. Note that just by loading the sound into Geometry Nodes, it is not played back. It needs to be added to e.g. the sequencer too. Pull Request: https://projects.blender.org/blender/blender/pulls/156247
This commit is contained in:
Jacques Lucke
parent
5215fb5978
commit
ccedfb3357
9 changed files with 877 additions and 5 deletions
|
|
@ -317,6 +317,7 @@ class NODE_MT_gn_input_base(node_add_menu.NodeMenu):
|
|||
self.draw_menu(layout, path="Input/Group")
|
||||
self.draw_menu(layout, path="Input/Import")
|
||||
self.draw_menu(layout, path="Input/Scene")
|
||||
self.draw_menu(layout, path="Input/Sound")
|
||||
|
||||
self.draw_assets_for_catalog(layout, self.bl_label)
|
||||
|
||||
|
|
@ -674,6 +675,15 @@ class NODE_MT_gn_utilities_text_base(node_add_menu.NodeMenu):
|
|||
self.draw_assets_for_catalog(layout, self.menu_path)
|
||||
|
||||
|
||||
class NODE_MT_gn_utilities_sound_base(node_add_menu.NodeMenu):
|
||||
bl_label = "Sound"
|
||||
menu_path = "Utilities/Sound"
|
||||
|
||||
def draw(self, context):
|
||||
layout = self.layout
|
||||
self.node_operator(layout, "GeometryNodeSampleSoundFrequencies")
|
||||
|
||||
|
||||
class NODE_MT_gn_texture_base(node_add_menu.NodeMenu):
|
||||
bl_label = "Texture"
|
||||
|
||||
|
|
@ -709,6 +719,7 @@ class NODE_MT_gn_utilities_base(node_add_menu.NodeMenu):
|
|||
self.draw_menu(layout, path="Utilities/List")
|
||||
self.draw_menu(layout, path="Utilities/Matrix")
|
||||
self.draw_menu(layout, path="Utilities/Rotation")
|
||||
self.draw_menu(layout, path="Utilities/Sound")
|
||||
layout.separator()
|
||||
self.node_operator(layout, "NodeImplicitConversion")
|
||||
self.for_each_element_zone(layout, label="For Each Element")
|
||||
|
|
@ -1115,6 +1126,7 @@ add_menus = {
|
|||
"NODE_MT_category_utilities_list": NODE_MT_gn_utilities_list_base,
|
||||
"NODE_MT_category_utilities_matrix": NODE_MT_gn_utilities_matrix_base,
|
||||
"NODE_MT_category_GEO_UTILITIES_DEPRECATED": NODE_MT_gn_utilities_deprecated_base,
|
||||
"NODE_MT_category_input_sound": NODE_MT_gn_utilities_sound_base,
|
||||
"NODE_MT_geometry_node_add_all": NODE_MT_gn_all_base,
|
||||
}
|
||||
add_menus = node_add_menu.generate_menus(
|
||||
|
|
@ -1133,6 +1145,7 @@ swap_menus = {
|
|||
"NODE_MT_gn_input_group_swap": NODE_MT_gn_input_group_base,
|
||||
"NODE_MT_gn_input_import_swap": NODE_MT_gn_input_import_base,
|
||||
"NODE_MT_gn_input_scene_swap": NODE_MT_gn_input_scene_base,
|
||||
"NODE_MT_gn_input_sound_swap": NODE_MT_gn_utilities_sound_base,
|
||||
"NODE_MT_gn_output_swap": NODE_MT_gn_output_base,
|
||||
"NODE_MT_gn_curve_swap": NODE_MT_gn_curve_base,
|
||||
"NODE_MT_gn_curve_read_swap": NODE_MT_gn_curve_read_base,
|
||||
|
|
|
|||
124
source/blender/blenkernel/BKE_sound_sample.hh
Normal file
124
source/blender/blenkernel/BKE_sound_sample.hh
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
/* SPDX-FileCopyrightText: 2025 Blender Authors
|
||||
*
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
|
||||
#include "BLI_array.hh"
|
||||
#include "BLI_cache_mutex.hh"
|
||||
#include "BLI_hash.hh"
|
||||
|
||||
#include "BKE_sound.hh"
|
||||
|
||||
namespace blender::bke {
|
||||
|
||||
/**
|
||||
* This class allows efficiently sampling an arbitrary frequency range at an arbitrary point in
|
||||
* time. This is achieved by caching the result of the fourier transform for various windows and
|
||||
* interpolating between the cached values.
|
||||
*
|
||||
* Used by the Sample Sound node in Geometry Nodes.
|
||||
*/
|
||||
class bSoundFrequencySampler {
|
||||
public:
|
||||
/**
|
||||
* Window function that is applied before computing the FFT. It avoids spectral leakage and
|
||||
* generally leads to better results. Using #Rectangular mode is the same as not using any window
|
||||
* function.
|
||||
*/
|
||||
enum class WindowFunction {
|
||||
Hann,
|
||||
Hamming,
|
||||
Blackman,
|
||||
Rectangular,
|
||||
};
|
||||
|
||||
enum class InterpolationMethod {
|
||||
/**
|
||||
* Fastest method but a resulting spectrum visualization may not be continuous, especially for
|
||||
* lower frequencies.
|
||||
*/
|
||||
Linear,
|
||||
/** Slower but produces a continuous spectrum. It may not be entirely smooth though. */
|
||||
CatmullRom,
|
||||
/** Yet slower but produces a continuous spectrum that is smooth. */
|
||||
BSpline,
|
||||
};
|
||||
|
||||
struct Key {
|
||||
WindowFunction window_function;
|
||||
/** This has to be a power of two. */
|
||||
int fft_size;
|
||||
/** If nullopt, the all channels are mixed. */
|
||||
std::optional<int> channel;
|
||||
|
||||
uint64_t hash() const
|
||||
{
|
||||
return get_default_hash(this->window_function, this->fft_size, this->channel.value_or(-1));
|
||||
}
|
||||
|
||||
friend bool operator==(const Key &a, const Key &b) = default;
|
||||
};
|
||||
|
||||
/** Precomputed weights for a specific window function and FFT size. */
|
||||
struct WindowWeights {
|
||||
Array<float> weights;
|
||||
float weights_sum;
|
||||
};
|
||||
|
||||
private:
|
||||
/** Cache for a single window. The frequencies are computed lazily when necessary. */
|
||||
struct WindowCache {
|
||||
mutable CacheMutex mutex;
|
||||
/**
|
||||
* Stores the amplitudes of the individual frequencies in this window using a prefix-sum. This
|
||||
* allows constant time lookup for a range of frequencies.
|
||||
*/
|
||||
mutable std::optional<Array<float, 0>> cumulative_amplitudes;
|
||||
};
|
||||
|
||||
AUD_Sound sound_;
|
||||
Key key_;
|
||||
/** Derived from the sound. */
|
||||
int samples_per_second_;
|
||||
/**
|
||||
* Determines the offset of one window to the next in samples.
|
||||
*
|
||||
* The larger the value, the fewer FFTs have to be computed resulting in faster playback at the
|
||||
* cost of resolution on the time domain. Small values also increase the memory usage.
|
||||
*/
|
||||
int window_cache_stride_;
|
||||
Array<WindowCache> window_caches_;
|
||||
|
||||
/** Cached weights of the selected window function. */
|
||||
const WindowWeights &window_weights_;
|
||||
|
||||
public:
|
||||
/** Construct a new sampler, prefer using #get_cached instead. */
|
||||
bSoundFrequencySampler(AUD_Sound sound, const Key &key);
|
||||
|
||||
/** Access a reusable frequency sampler for the given sound. */
|
||||
static const bSoundFrequencySampler *get_cached(const bSound &sound, const Key &key);
|
||||
|
||||
/** Sample the amplitude a the given time and frequency range. */
|
||||
float sample(float time,
|
||||
float low,
|
||||
float high,
|
||||
InterpolationMethod time_interpolation,
|
||||
InterpolationMethod frequency_interpolation) const;
|
||||
|
||||
private:
|
||||
float sample_frequency_range_in_window(int window_i,
|
||||
float low,
|
||||
float high,
|
||||
InterpolationMethod method) const;
|
||||
float sample_cumulative_frequency(Span<float> window_values,
|
||||
float frequency,
|
||||
InterpolationMethod method) const;
|
||||
std::optional<Span<float>> ensure_window_cache(int window_i) const;
|
||||
std::optional<Array<float>> compute_fft(int start_sample) const;
|
||||
};
|
||||
|
||||
} // namespace blender::bke
|
||||
|
|
@ -504,6 +504,7 @@ set(SRC
|
|||
BKE_shrinkwrap.hh
|
||||
BKE_softbody.h
|
||||
BKE_sound.hh
|
||||
BKE_sound_sample.hh
|
||||
BKE_sound_types.hh
|
||||
BKE_speaker.hh
|
||||
BKE_studiolight.h
|
||||
|
|
|
|||
|
|
@ -12,9 +12,14 @@
|
|||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <mutex>
|
||||
#include <numeric>
|
||||
#include <optional>
|
||||
#include <thread>
|
||||
|
||||
#ifdef WITH_FFTW3
|
||||
# include <fftw3.h>
|
||||
#endif
|
||||
|
||||
#include "MEM_guardedalloc.h"
|
||||
|
||||
#include "BLI_build_config.h"
|
||||
|
|
@ -84,6 +89,7 @@
|
|||
#include "BKE_packedFile.hh"
|
||||
#include "BKE_scene_runtime.hh"
|
||||
#include "BKE_sound.hh"
|
||||
#include "BKE_sound_sample.hh"
|
||||
|
||||
#include "DEG_depsgraph.hh"
|
||||
#include "DEG_depsgraph_query.hh"
|
||||
|
|
@ -93,6 +99,8 @@
|
|||
#include "SEQ_sequencer.hh"
|
||||
#include "SEQ_sound.hh"
|
||||
|
||||
#include "BLI_concurrent_map.hh"
|
||||
|
||||
#include "CLG_log.h"
|
||||
|
||||
namespace blender {
|
||||
|
|
@ -108,6 +116,9 @@ enum class SoundTags {
|
|||
};
|
||||
ENUM_OPERATORS(SoundTags);
|
||||
|
||||
using bSoundFrequencySamplerMap =
|
||||
ConcurrentMap<bSoundFrequencySampler::Key, std::shared_ptr<bSoundFrequencySampler>>;
|
||||
|
||||
struct SoundRuntime {
|
||||
AUD_Sound handle;
|
||||
AUD_Sound cache;
|
||||
|
|
@ -121,6 +132,9 @@ struct SoundRuntime {
|
|||
* save/restore a pointer. */
|
||||
Vector<float> *waveform = nullptr;
|
||||
SoundTags tags = SoundTags::None;
|
||||
|
||||
/** Caches frequency samplers for this sound. */
|
||||
bSoundFrequencySamplerMap samplers;
|
||||
};
|
||||
|
||||
} // namespace bke
|
||||
|
|
@ -2098,4 +2112,358 @@ const Vector<float> *BKE_sound_runtime_get_waveform(const bSound *sound)
|
|||
return sound->runtime->waveform;
|
||||
}
|
||||
|
||||
namespace bke {
|
||||
|
||||
const bSoundFrequencySampler *bSoundFrequencySampler::get_cached(const bSound &sound,
|
||||
const Key &key)
|
||||
{
|
||||
#ifdef WITH_AUDASPACE
|
||||
{
|
||||
/* Fast common case when the sampler has been created already. */
|
||||
bSoundFrequencySamplerMap::ConstAccessor accessor;
|
||||
if (sound.runtime->samplers.lookup(accessor, key)) {
|
||||
return accessor->second.get();
|
||||
}
|
||||
}
|
||||
AUD_Sound sound_handle = sound.runtime->handle;
|
||||
if (!sound_handle) {
|
||||
/* Maybe try to load the sound in this case instead of relying on cache. */
|
||||
return nullptr;
|
||||
}
|
||||
/* Slower case when the sampler is newly created. */
|
||||
bSoundFrequencySamplerMap::MutableAccessor accessor;
|
||||
if (sound.runtime->samplers.add(accessor, key)) {
|
||||
if (key.channel.has_value()) {
|
||||
const SoundInfo info = sound_info_get(sound_handle);
|
||||
const int channel = *key.channel;
|
||||
if (channel < 0 || channel >= info.specs.channels) {
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
accessor->second = std::make_shared<bSoundFrequencySampler>(sound_handle, key);
|
||||
}
|
||||
return accessor->second.get();
|
||||
#else
|
||||
UNUSED_VARS(sound, key);
|
||||
return nullptr;
|
||||
#endif
|
||||
}
|
||||
|
||||
static bSoundFrequencySampler::WindowWeights compute_window_function_weights(
|
||||
const bSoundFrequencySampler::WindowFunction window, const int size)
|
||||
{
|
||||
Array<float> weights(size);
|
||||
switch (window) {
|
||||
case bSoundFrequencySampler::WindowFunction::Hann: {
|
||||
for (const int i : IndexRange(size)) {
|
||||
weights[i] = 0.5f - 0.5f * math::cos((2.0f * std::numbers::pi * i) / (size - 1));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case bSoundFrequencySampler::WindowFunction::Hamming: {
|
||||
for (const int i : IndexRange(size)) {
|
||||
weights[i] = 0.54f - 0.46f * math::cos((2.0f * std::numbers::pi * i) / (size - 1));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case bSoundFrequencySampler::WindowFunction::Blackman: {
|
||||
for (const int i : IndexRange(size)) {
|
||||
weights[i] = 0.42f - 0.5f * math::cos((2.0f * std::numbers::pi * i) / (size - 1)) +
|
||||
0.08f * math::cos((4.0f * std::numbers::pi * i) / (size - 1));
|
||||
}
|
||||
break;
|
||||
}
|
||||
case bSoundFrequencySampler::WindowFunction::Rectangular: {
|
||||
weights.fill(1.0f);
|
||||
break;
|
||||
}
|
||||
}
|
||||
const float sum = std::accumulate(weights.begin(), weights.end(), 0.0f);
|
||||
return bSoundFrequencySampler::WindowWeights{std::move(weights), sum};
|
||||
}
|
||||
|
||||
/** Caches the window function weights so that each combination is only computed once. */
|
||||
static const bSoundFrequencySampler::WindowWeights &get_window_function_weights(
|
||||
const bSoundFrequencySampler::WindowFunction window, const int size)
|
||||
{
|
||||
static Mutex mutex;
|
||||
static Map<std::pair<bSoundFrequencySampler::WindowFunction, int>,
|
||||
std::unique_ptr<bSoundFrequencySampler::WindowWeights>>
|
||||
map;
|
||||
std::lock_guard lock{mutex};
|
||||
return *map.lookup_or_add_cb({window, size}, [&]() {
|
||||
return std::make_unique<bSoundFrequencySampler::WindowWeights>(
|
||||
compute_window_function_weights(window, size));
|
||||
});
|
||||
}
|
||||
|
||||
bSoundFrequencySampler::bSoundFrequencySampler(AUD_Sound sound, const Key &key)
|
||||
: sound_(sound),
|
||||
key_(key),
|
||||
window_weights_(get_window_function_weights(key.window_function, key.fft_size))
|
||||
{
|
||||
#ifdef WITH_AUDASPACE
|
||||
const SoundInfo info = bke::sound_info_get(sound_);
|
||||
samples_per_second_ = info.specs.samplerate;
|
||||
/* This could be a parameter but a single fixed value seems fine for now and makes caching much
|
||||
* simpler. */
|
||||
window_cache_stride_ = key.fft_size / 8;
|
||||
const int window_caches_num = std::ceil(info.length * info.specs.samplerate /
|
||||
window_cache_stride_);
|
||||
window_caches_.reinitialize(window_caches_num);
|
||||
#else
|
||||
UNUSED_VARS(sound, key);
|
||||
BLI_assert_unreachable();
|
||||
#endif
|
||||
}
|
||||
|
||||
std::optional<Array<float>> bSoundFrequencySampler::compute_fft(const int start_sample) const
|
||||
{
|
||||
/* Since the result of the dft algorithm is symmetric in this case, only the first half is
|
||||
* computed. */
|
||||
const int frequencies_num = key_.fft_size / 2;
|
||||
|
||||
#if defined(WITH_AUDASPACE) && defined(WITH_FFTW3)
|
||||
/* Read some extra samples before the ones we are actually interested in here. This is done
|
||||
* because the #read function may sometimes give invalid data for the first samples. */
|
||||
const int warmup_samples = std::min(2000, start_sample);
|
||||
|
||||
/* Prepare the reader. */
|
||||
std::shared_ptr<aud::IReader> reader = sound_->createReader();
|
||||
const aud::Specs specs = reader->getSpecs();
|
||||
const int channels_num = specs.channels;
|
||||
|
||||
/* Read the raw samples from the audio stream. */
|
||||
Array<float> read_buffer_extra((key_.fft_size + warmup_samples) * channels_num);
|
||||
bool is_end_of_stream = false;
|
||||
int length = key_.fft_size + warmup_samples;
|
||||
reader->seek(std::max(start_sample - warmup_samples, 0));
|
||||
reader->read(length, is_end_of_stream, read_buffer_extra.data());
|
||||
const Span<float> read_buffer = read_buffer_extra.as_span().drop_front(warmup_samples *
|
||||
channels_num);
|
||||
const int read_length = read_buffer.size() / channels_num;
|
||||
|
||||
/* Pull out the samples for the requested channel(s). */
|
||||
Array<float> buffer(key_.fft_size, 0.0f);
|
||||
if (key_.channel.has_value()) {
|
||||
const int channel = *key_.channel;
|
||||
if (channel < 0 || channel >= channels_num) {
|
||||
return std::nullopt;
|
||||
}
|
||||
for (const int i : IndexRange(read_length)) {
|
||||
buffer[i] = read_buffer[i * channels_num + channel];
|
||||
}
|
||||
}
|
||||
else {
|
||||
for (const int i : IndexRange(read_length)) {
|
||||
for (const int c : IndexRange(channels_num)) {
|
||||
buffer[i] += read_buffer[i * channels_num + c];
|
||||
}
|
||||
buffer[i] /= channels_num;
|
||||
}
|
||||
}
|
||||
|
||||
/* Apply window function which avoids spectral leakage (depending on the function). */
|
||||
for (const int i : IndexRange(read_length)) {
|
||||
buffer[i] *= window_weights_.weights[i];
|
||||
}
|
||||
|
||||
/* Set up the fftw plan. */
|
||||
fftwf_complex *fftwf_buffer = static_cast<fftwf_complex *>(
|
||||
fftwf_malloc(sizeof(fftwf_complex) * (frequencies_num + 1)));
|
||||
fftwf_plan plan = fftwf_plan_dft_r2c_1d(
|
||||
key_.fft_size, buffer.data(), fftwf_buffer, FFTW_ESTIMATE);
|
||||
BLI_SCOPED_DEFER([&]() {
|
||||
fftwf_destroy_plan(plan);
|
||||
fftwf_free(fftwf_buffer);
|
||||
});
|
||||
|
||||
/* Actually perform the fourier transform. */
|
||||
fftwf_execute(plan);
|
||||
|
||||
/* Compute the amplitudes of the frequencies. */
|
||||
Array<float> frequency_amplitudes(frequencies_num);
|
||||
/* The scaling factor is applied so that changing the fft size or window function does not affect
|
||||
* the magnitude of the result. */
|
||||
const float scaling_factor = 1.0f / window_weights_.weights_sum;
|
||||
for (const int i : IndexRange(frequencies_num)) {
|
||||
const fftwf_complex &c = fftwf_buffer[i];
|
||||
/* Take real and imaginary parts into account which correspond to the sin and cos component of
|
||||
* the frequency. */
|
||||
frequency_amplitudes[i] = sqrt(pow2f(c[0]) + pow2f(c[1])) * scaling_factor;
|
||||
}
|
||||
|
||||
return frequency_amplitudes;
|
||||
#else
|
||||
UNUSED_VARS(start_sample);
|
||||
return Array<float>(frequencies_num, 0.0f);
|
||||
#endif
|
||||
}
|
||||
|
||||
static float catmul_rom_interpolation(
|
||||
const float p0, const float p1, const float p2, const float p3, const float t)
|
||||
{
|
||||
const float t2 = t * t;
|
||||
const float t3 = t2 * t;
|
||||
|
||||
const float value = 0.5 * ((2 * p1) + (-p0 + p2) * t + (2 * p0 - 5 * p1 + 4 * p2 - p3) * t2 +
|
||||
(-p0 + 3 * p1 - 3 * p2 + p3) * t3);
|
||||
return value;
|
||||
}
|
||||
|
||||
static float bspline_interpolation(
|
||||
const float p0, const float p1, const float p2, const float p3, const float t)
|
||||
{
|
||||
const float t2 = t * t;
|
||||
const float t3 = t2 * t;
|
||||
|
||||
const float b0 = (1.0f - t) * (1.0f - t) * (1.0f - t) / 6.0f;
|
||||
const float b1 = (3.0f * t3 - 6.0f * t2 + 4.0f) / 6.0f;
|
||||
const float b2 = (-3.0f * t3 + 3.0f * t2 + 3.0f * t + 1.0f) / 6.0f;
|
||||
const float b3 = t3 / 6.0f;
|
||||
|
||||
return (b0 * p0 + b1 * p1 + b2 * p2 + b3 * p3);
|
||||
}
|
||||
|
||||
float bSoundFrequencySampler::sample(const float time,
|
||||
const float low,
|
||||
const float high,
|
||||
const InterpolationMethod time_interpolation,
|
||||
const InterpolationMethod frequency_interpolation) const
|
||||
{
|
||||
if (low >= high) {
|
||||
return 0.0f;
|
||||
}
|
||||
const float i_float = std::max(
|
||||
0.0f, (time * samples_per_second_ - key_.fft_size / 2) / window_cache_stride_);
|
||||
const int i_pre = floorf(i_float);
|
||||
const int i_post = i_pre + 1;
|
||||
const float t = fractf(i_float);
|
||||
|
||||
switch (time_interpolation) {
|
||||
case InterpolationMethod::Linear: {
|
||||
const float v_prev = this->sample_frequency_range_in_window(
|
||||
i_pre, low, high, frequency_interpolation);
|
||||
const float v_next = this->sample_frequency_range_in_window(
|
||||
i_post, low, high, frequency_interpolation);
|
||||
|
||||
return math::interpolate(v_prev, v_next, t);
|
||||
}
|
||||
case InterpolationMethod::CatmullRom: {
|
||||
const int i_pre2 = i_pre - 1;
|
||||
const int i_post2 = i_post + 1;
|
||||
|
||||
const float p0 = this->sample_frequency_range_in_window(
|
||||
i_pre2, low, high, frequency_interpolation);
|
||||
const float p1 = this->sample_frequency_range_in_window(
|
||||
i_pre, low, high, frequency_interpolation);
|
||||
const float p2 = this->sample_frequency_range_in_window(
|
||||
i_post, low, high, frequency_interpolation);
|
||||
const float p3 = this->sample_frequency_range_in_window(
|
||||
i_post2, low, high, frequency_interpolation);
|
||||
|
||||
return catmul_rom_interpolation(p0, p1, p2, p3, t);
|
||||
}
|
||||
default:
|
||||
case InterpolationMethod::BSpline: {
|
||||
const int i_pre2 = i_pre - 1;
|
||||
const int i_post2 = i_post + 1;
|
||||
|
||||
const float p0 = this->sample_frequency_range_in_window(
|
||||
i_pre2, low, high, frequency_interpolation);
|
||||
const float p1 = this->sample_frequency_range_in_window(
|
||||
i_pre, low, high, frequency_interpolation);
|
||||
const float p2 = this->sample_frequency_range_in_window(
|
||||
i_post, low, high, frequency_interpolation);
|
||||
const float p3 = this->sample_frequency_range_in_window(
|
||||
i_post2, low, high, frequency_interpolation);
|
||||
|
||||
return bspline_interpolation(p0, p1, p2, p3, t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
float bSoundFrequencySampler::sample_frequency_range_in_window(int window_i,
|
||||
float low,
|
||||
float high,
|
||||
InterpolationMethod method) const
|
||||
{
|
||||
const std::optional<Span<float>> window_opt = this->ensure_window_cache(window_i);
|
||||
if (!window_opt.has_value()) {
|
||||
return 0.0f;
|
||||
}
|
||||
const Span<float> window_values = *window_opt;
|
||||
const float cumulative_low = this->sample_cumulative_frequency(window_values, low, method);
|
||||
const float cumulative_high = this->sample_cumulative_frequency(window_values, high, method);
|
||||
return cumulative_high - cumulative_low;
|
||||
}
|
||||
|
||||
float bSoundFrequencySampler::sample_cumulative_frequency(const Span<float> window_values,
|
||||
const float frequency,
|
||||
const InterpolationMethod method) const
|
||||
{
|
||||
const int max_i = window_values.size() - 1;
|
||||
const float i_float = frequency * key_.fft_size / samples_per_second_;
|
||||
const int i_pre = std::clamp<int>(std::floor(i_float), 0, max_i);
|
||||
const int i_post = std::min(i_pre + 1, max_i);
|
||||
const float t = fractf(i_float);
|
||||
|
||||
switch (method) {
|
||||
case InterpolationMethod::Linear: {
|
||||
return math::interpolate(window_values[i_pre], window_values[i_post], t);
|
||||
}
|
||||
case InterpolationMethod::CatmullRom: {
|
||||
const int i_pre2 = std::max(i_pre - 1, 0);
|
||||
const int i_post2 = std::min(i_post + 1, max_i);
|
||||
|
||||
const float p0 = window_values[i_pre2];
|
||||
const float p1 = window_values[i_pre];
|
||||
const float p2 = window_values[i_post];
|
||||
const float p3 = window_values[i_post2];
|
||||
|
||||
return catmul_rom_interpolation(p0, p1, p2, p3, t);
|
||||
}
|
||||
case InterpolationMethod::BSpline: {
|
||||
const int i_pre2 = std::max(i_pre - 1, 0);
|
||||
const int i_post2 = std::min(i_post + 1, max_i);
|
||||
|
||||
const float p0 = window_values[i_pre2];
|
||||
const float p1 = window_values[i_pre];
|
||||
const float p2 = window_values[i_post];
|
||||
const float p3 = window_values[i_post2];
|
||||
|
||||
return bspline_interpolation(p0, p1, p2, p3, t);
|
||||
}
|
||||
}
|
||||
return 0.0f;
|
||||
}
|
||||
|
||||
std::optional<Span<float>> bSoundFrequencySampler::ensure_window_cache(int window_i) const
|
||||
{
|
||||
/* Clamp window so that too low or high indices get mapped to the closest valid index. */
|
||||
window_i = std::clamp<int>(window_i, 0, window_caches_.size() - 1);
|
||||
|
||||
const WindowCache &window = window_caches_[window_i];
|
||||
/* Compute the FFT of that window if that wasn't done already. */
|
||||
window.mutex.ensure([&]() {
|
||||
std::optional<Array<float>> fft_array = this->compute_fft(window_i * window_cache_stride_);
|
||||
if (!fft_array.has_value()) {
|
||||
return;
|
||||
}
|
||||
/* Compute prefix sum for the fft values. */
|
||||
const Span<float> fft_values = fft_array->as_span();
|
||||
window.cumulative_amplitudes.emplace(fft_values.size() + 1);
|
||||
float sum = 0.0f;
|
||||
for (const int i : fft_array->index_range()) {
|
||||
const float value = std::abs(fft_values[i]);
|
||||
(*window.cumulative_amplitudes)[i] = sum;
|
||||
sum += value;
|
||||
}
|
||||
window.cumulative_amplitudes->last() = sum;
|
||||
});
|
||||
return window.cumulative_amplitudes;
|
||||
}
|
||||
|
||||
} // namespace bke
|
||||
|
||||
} // namespace blender
|
||||
|
|
|
|||
|
|
@ -1004,7 +1004,7 @@ static const float std_node_socket_colors[][4] = {
|
|||
{0, 0, 0, 1}, /* SOCK_SCENE */
|
||||
{0, 0, 0, 1}, /* SOCK_TEXT_ID */
|
||||
{0, 0, 0, 1}, /* SOCK_MASK */
|
||||
{0, 0, 0, 1}, /* SOCK_SOUND */
|
||||
{0.39, 0.34, 0.26, 1}, /* SOCK_SOUND */
|
||||
{0.36, 0.47, 0.61, 1.0}, /* SOCK_INT_VECTOR */
|
||||
};
|
||||
|
||||
|
|
@ -1365,8 +1365,7 @@ static void std_node_socket_draw(
|
|||
case SOCK_MATERIAL:
|
||||
case SOCK_SCENE:
|
||||
case SOCK_TEXT_ID:
|
||||
case SOCK_MASK:
|
||||
case SOCK_SOUND: {
|
||||
case SOCK_MASK: {
|
||||
if (optional_label) {
|
||||
layout->prop(ptr,
|
||||
RNA_struct_find_property(ptr, "default_value"),
|
||||
|
|
@ -1386,7 +1385,18 @@ static void std_node_socket_draw(
|
|||
label,
|
||||
ICON_NONE);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case SOCK_SOUND: {
|
||||
if (optional_label) {
|
||||
template_id(layout, C, ptr, "default_value", nullptr, "SOUND_OT_open", nullptr);
|
||||
}
|
||||
else {
|
||||
/* 0.3 is consistent with image sockets. */
|
||||
ui::Layout *row = &layout->split(0.3f, false);
|
||||
row->label(label, ICON_NONE);
|
||||
template_id(row, C, ptr, "default_value", nullptr, "SOUND_OT_open", nullptr);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case SOCK_FONT: {
|
||||
|
|
|
|||
|
|
@ -10566,6 +10566,7 @@ static void rna_def_nodes(BlenderRNA *brna)
|
|||
define("GeometryNode", "GeometryNodeSampleIndex", def_geo_sample_index);
|
||||
define("GeometryNode", "GeometryNodeSampleNearest");
|
||||
define("GeometryNode", "GeometryNodeSampleNearestSurface");
|
||||
define("GeometryNode", "GeometryNodeSampleSoundFrequencies");
|
||||
define("GeometryNode", "GeometryNodeSampleUVSurface");
|
||||
define("GeometryNode", "GeometryNodeScaleElements");
|
||||
define("GeometryNode", "GeometryNodeScaleInstances");
|
||||
|
|
|
|||
|
|
@ -219,6 +219,7 @@ set(SRC
|
|||
nodes/node_geo_sample_index.cc
|
||||
nodes/node_geo_sample_nearest.cc
|
||||
nodes/node_geo_sample_nearest_surface.cc
|
||||
nodes/node_geo_sample_sound_frequencies.cc
|
||||
nodes/node_geo_sample_uv_surface.cc
|
||||
nodes/node_geo_scale_elements.cc
|
||||
nodes/node_geo_scale_instances.cc
|
||||
|
|
|
|||
|
|
@ -139,7 +139,7 @@ static bool geometry_node_tree_socket_type_valid(bke::bNodeTreeType * /*treetype
|
|||
SOCK_IMAGE,
|
||||
SOCK_MATERIAL,
|
||||
SOCK_MENU) ||
|
||||
ELEM(socket_type->type, SOCK_BUNDLE, SOCK_CLOSURE, SOCK_FONT));
|
||||
ELEM(socket_type->type, SOCK_BUNDLE, SOCK_CLOSURE, SOCK_FONT, SOCK_SOUND));
|
||||
}
|
||||
|
||||
void register_node_tree_type_geo()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,354 @@
|
|||
/* SPDX-FileCopyrightText: 2026 Blender Authors
|
||||
*
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later */
|
||||
|
||||
#include "BKE_sound_sample.hh"
|
||||
|
||||
#include "NOD_socket_usage_inference.hh"
|
||||
|
||||
#include "node_geometry_util.hh"
|
||||
|
||||
namespace blender::nodes::node_geo_sample_sound_frequencies_cc {
|
||||
|
||||
enum class FFTSize {
|
||||
_128 = 128,
|
||||
_256 = 256,
|
||||
_512 = 512,
|
||||
_1024 = 1024,
|
||||
_2048 = 2048,
|
||||
_4096 = 4096,
|
||||
_8192 = 8192,
|
||||
_16384 = 16384,
|
||||
_32768 = 32768,
|
||||
};
|
||||
|
||||
enum class WindowFunction {
|
||||
Hann = 0,
|
||||
Hamming = 1,
|
||||
Blackman = 2,
|
||||
Rectangular = 3,
|
||||
};
|
||||
|
||||
static const EnumPropertyItem fft_size_items[] = {
|
||||
{int(FFTSize::_128), "128", 0, "128", ""},
|
||||
{int(FFTSize::_256), "256", 0, "256", ""},
|
||||
{int(FFTSize::_512), "512", 0, "512", ""},
|
||||
{int(FFTSize::_1024), "1024", 0, "1024", ""},
|
||||
{int(FFTSize::_2048), "2048", 0, "2048", ""},
|
||||
{int(FFTSize::_4096), "4096", 0, "4096", ""},
|
||||
{int(FFTSize::_8192), "8192", 0, "8192", ""},
|
||||
{int(FFTSize::_16384), "16384", 0, "16384", ""},
|
||||
{int(FFTSize::_32768), "32768", 0, "32768", ""},
|
||||
{},
|
||||
};
|
||||
|
||||
static const EnumPropertyItem window_function_items[] = {
|
||||
{int(WindowFunction::Hann), "Hann", 0, "Hann", ""},
|
||||
{int(WindowFunction::Hamming), "Hamming", 0, "Hamming", ""},
|
||||
{int(WindowFunction::Blackman), "Blackman", 0, "Blackman", ""},
|
||||
{int(WindowFunction::Rectangular),
|
||||
"Rectangular",
|
||||
0,
|
||||
"Rectangular",
|
||||
"Equivalent to having no window function"},
|
||||
{},
|
||||
};
|
||||
|
||||
static void node_declare(NodeDeclarationBuilder &b)
|
||||
{
|
||||
b.use_custom_socket_order();
|
||||
b.allow_any_socket_order();
|
||||
|
||||
b.add_output<decl::Float>("Amplitude"_ustr)
|
||||
.reference_pass_all()
|
||||
.description("Sum of amplitudes of the frequencies in the given range")
|
||||
.structure_type(StructureType::Dynamic);
|
||||
b.add_input<decl::Sound>("Sound"_ustr).optional_label().description("Sound to sample");
|
||||
b.add_input<decl::Float>("Time"_ustr)
|
||||
.subtype(PROP_TIME_ABSOLUTE)
|
||||
.supports_field()
|
||||
.structure_type(StructureType::Dynamic)
|
||||
.description("Time in seconds of the sound to sample at");
|
||||
b.add_input<decl::Bool>("All Channels"_ustr)
|
||||
.default_value(true)
|
||||
.supports_field()
|
||||
.structure_type(StructureType::Dynamic)
|
||||
.description("Mix all channels before sampling the sound (e.g. stereo to mono)");
|
||||
b.add_input<decl::Int>("Channel"_ustr)
|
||||
.min(0)
|
||||
.usage_inference(
|
||||
[](const socket_usage_inference::SocketUsageParams ¶ms) -> std::optional<bool> {
|
||||
if (const std::optional<bool> any_output_used = params.any_output_is_used()) {
|
||||
if (!*any_output_used) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else {
|
||||
return std::nullopt;
|
||||
}
|
||||
const std::optional<bool> all_channels =
|
||||
params.get_input("All Channels"_ustr).get_if_primitive<bool>();
|
||||
if (!all_channels.has_value()) {
|
||||
return true;
|
||||
}
|
||||
return !*all_channels;
|
||||
})
|
||||
.supports_field()
|
||||
.structure_type(StructureType::Dynamic)
|
||||
.description("The channel to sample unless 'All Channels' is checked");
|
||||
b.add_input<decl::Float>("Low"_ustr)
|
||||
.subtype(PROP_FREQUENCY)
|
||||
.default_value(0.0f)
|
||||
.min(0.0f)
|
||||
.supports_field()
|
||||
.structure_type(StructureType::Dynamic)
|
||||
.description("Lower bound of the sampled frequency range");
|
||||
b.add_input<decl::Float>("High"_ustr)
|
||||
.subtype(PROP_FREQUENCY)
|
||||
.default_value(10'000.0f)
|
||||
.min(0.0f)
|
||||
.supports_field()
|
||||
.structure_type(StructureType::Dynamic)
|
||||
.description("Upper bound of the sampled frequency range");
|
||||
|
||||
{
|
||||
auto &p = b.add_panel("FFT"_ustr)
|
||||
.default_closed(true)
|
||||
.description("Configure details of the fourier transformation");
|
||||
p.add_input<decl::Menu>("FFT Size"_ustr)
|
||||
.static_items(fft_size_items)
|
||||
.default_value(FFTSize::_4096)
|
||||
.optional_label()
|
||||
.description(
|
||||
"Number of samples to process in the discrete fourier transformation at once. Higher "
|
||||
"values have higher frequency but lower time resolution and vice versa");
|
||||
p.add_input<decl::Menu>("Window Function"_ustr)
|
||||
.static_items(window_function_items)
|
||||
.default_value(WindowFunction::Hann)
|
||||
.optional_label()
|
||||
.description(
|
||||
"Applies a tapering function to the windowed samples to minimize discontinuities at "
|
||||
"the edges, improving frequency resolution and reducing artifacts");
|
||||
}
|
||||
}
|
||||
|
||||
class SampleSoundFunction : public mf::MultiFunction {
|
||||
private:
|
||||
const bSound &sound_;
|
||||
const int fft_size_;
|
||||
const bke::bSoundFrequencySampler::WindowFunction window_function_;
|
||||
|
||||
public:
|
||||
SampleSoundFunction(bSound &sound,
|
||||
const int fft_size,
|
||||
const bke::bSoundFrequencySampler::WindowFunction window_function)
|
||||
: sound_(sound), fft_size_(fft_size), window_function_(window_function)
|
||||
{
|
||||
static const mf::Signature signature = []() {
|
||||
mf::Signature signature;
|
||||
mf::SignatureBuilder builder("Sample Sound", signature);
|
||||
builder.single_input<float>("Time");
|
||||
builder.single_input<bool>("All Channels");
|
||||
builder.single_input<int>("Channel");
|
||||
builder.single_input<float>("Low");
|
||||
builder.single_input<float>("High");
|
||||
builder.single_output<float>("Amplitude");
|
||||
return signature;
|
||||
}();
|
||||
this->set_signature(&signature);
|
||||
BLI_assert(is_power_of_2(fft_size_));
|
||||
}
|
||||
|
||||
void call(const IndexMask &mask, mf::Params params, mf::Context /*context*/) const override
|
||||
{
|
||||
const VArray<float> × = params.readonly_single_input<float>(0, "Time");
|
||||
const VArray<bool> &all_channels_varray = params.readonly_single_input<bool>(1,
|
||||
"All Channels");
|
||||
const VArray<int> &channels = params.readonly_single_input<int>(2, "Channel");
|
||||
const VArray<float> &lows = params.readonly_single_input<float>(3, "Low");
|
||||
const VArray<float> &highs = params.readonly_single_input<float>(4, "High");
|
||||
MutableSpan<float> amplitudes = params.uninitialized_single_output<float>(5, "Amplitude");
|
||||
|
||||
const std::optional<bool> all_channels_value = all_channels_varray.get_if_single();
|
||||
const std::optional<float> channel_value = channels.get_if_single();
|
||||
const bool constant_channel = all_channels_value == true ||
|
||||
(all_channels_value.has_value() && channel_value.has_value());
|
||||
|
||||
const auto time_interpolation = bke::bSoundFrequencySampler::InterpolationMethod::BSpline;
|
||||
const auto frequency_interpolation = bke::bSoundFrequencySampler::InterpolationMethod::BSpline;
|
||||
|
||||
/* Optimize the case when all indices sample the same channel. */
|
||||
if (constant_channel) {
|
||||
bke::bSoundFrequencySampler::Key key;
|
||||
key.window_function = window_function_;
|
||||
key.fft_size = fft_size_;
|
||||
key.channel = *all_channels_value ? std::nullopt : channel_value;
|
||||
const bke::bSoundFrequencySampler *sampler = bke::bSoundFrequencySampler::get_cached(sound_,
|
||||
key);
|
||||
if (!sampler) {
|
||||
index_mask::masked_fill(amplitudes, 0.0f, mask);
|
||||
return;
|
||||
}
|
||||
mask.foreach_index([&](const int i) {
|
||||
const float time = times[i];
|
||||
const float low = lows[i];
|
||||
const float high = highs[i];
|
||||
const float amplitude = sampler->sample(
|
||||
time, low, high, time_interpolation, frequency_interpolation);
|
||||
amplitudes[i] = amplitude;
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
/* Sort indices by channel, optimizing for the common case that the channel number is typically
|
||||
* very low. */
|
||||
constexpr int channel_array_size = 6;
|
||||
std::array<Vector<int>, channel_array_size> indices_by_channel;
|
||||
MultiValueMap<int, int> indices_with_different_channel;
|
||||
Vector<int> indices_with_all_channels;
|
||||
mask.foreach_index([&](const int i) {
|
||||
const bool all_channels = all_channels_varray[i];
|
||||
if (all_channels) {
|
||||
indices_with_all_channels.append(i);
|
||||
}
|
||||
else {
|
||||
const int channel = channels[i];
|
||||
if (channel >= 0 && channel < channel_array_size) {
|
||||
indices_by_channel[channel].append(i);
|
||||
}
|
||||
else {
|
||||
indices_with_different_channel.add(channel, i);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
/* Actually sample the indices by channel. */
|
||||
auto sample_indices_in_channel = [&](const Span<int> indices,
|
||||
const std::optional<int> channel) {
|
||||
if (indices.is_empty()) {
|
||||
return;
|
||||
}
|
||||
bke::bSoundFrequencySampler::Key key;
|
||||
key.window_function = window_function_;
|
||||
key.fft_size = fft_size_;
|
||||
key.channel = channel;
|
||||
const bke::bSoundFrequencySampler *sampler = bke::bSoundFrequencySampler::get_cached(sound_,
|
||||
key);
|
||||
if (!sampler) {
|
||||
amplitudes.fill_indices(indices, 0.0f);
|
||||
return;
|
||||
}
|
||||
for (const int i : indices) {
|
||||
const float time = times[i];
|
||||
const float low = lows[i];
|
||||
const float high = highs[i];
|
||||
const float amplitude = sampler->sample(
|
||||
time, low, high, time_interpolation, frequency_interpolation);
|
||||
amplitudes[i] = amplitude;
|
||||
}
|
||||
};
|
||||
|
||||
sample_indices_in_channel(indices_with_all_channels, std::nullopt);
|
||||
for (const int channel : IndexRange(channel_array_size)) {
|
||||
sample_indices_in_channel(indices_by_channel[channel], channel);
|
||||
}
|
||||
for (const auto item : indices_with_different_channel.items()) {
|
||||
sample_indices_in_channel(item.value, item.key);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
static int to_fft_size_int(const FFTSize fft_size)
|
||||
{
|
||||
switch (fft_size) {
|
||||
case FFTSize::_128:
|
||||
return 128;
|
||||
case FFTSize::_256:
|
||||
return 256;
|
||||
case FFTSize::_512:
|
||||
return 512;
|
||||
case FFTSize::_1024:
|
||||
return 1024;
|
||||
case FFTSize::_2048:
|
||||
return 2048;
|
||||
case FFTSize::_4096:
|
||||
return 4096;
|
||||
case FFTSize::_8192:
|
||||
return 8192;
|
||||
case FFTSize::_16384:
|
||||
return 16384;
|
||||
case FFTSize::_32768:
|
||||
return 32768;
|
||||
}
|
||||
return 4096;
|
||||
}
|
||||
|
||||
static bke::bSoundFrequencySampler::WindowFunction to_window_function(
|
||||
const WindowFunction window_function)
|
||||
{
|
||||
switch (window_function) {
|
||||
case WindowFunction::Hann:
|
||||
return bke::bSoundFrequencySampler::WindowFunction::Hann;
|
||||
case WindowFunction::Hamming:
|
||||
return bke::bSoundFrequencySampler::WindowFunction::Hamming;
|
||||
case WindowFunction::Blackman:
|
||||
return bke::bSoundFrequencySampler::WindowFunction::Blackman;
|
||||
case WindowFunction::Rectangular:
|
||||
return bke::bSoundFrequencySampler::WindowFunction::Rectangular;
|
||||
}
|
||||
return bke::bSoundFrequencySampler::WindowFunction::Hann;
|
||||
}
|
||||
|
||||
static void node_geo_exec(GeoNodeExecParams params)
|
||||
{
|
||||
bSound *sound = params.extract_input<bSound *>("Sound"_ustr);
|
||||
if (!sound) {
|
||||
params.set_default_remaining_outputs();
|
||||
return;
|
||||
}
|
||||
|
||||
const FFTSize fft_size = params.extract_input<FFTSize>("FFT Size"_ustr);
|
||||
const WindowFunction window_function = params.extract_input<WindowFunction>(
|
||||
"Window Function"_ustr);
|
||||
|
||||
SocketValueVariant times = params.extract_input<SocketValueVariant>("Time"_ustr);
|
||||
SocketValueVariant all_channels = params.extract_input<SocketValueVariant>("All Channels"_ustr);
|
||||
SocketValueVariant channels = params.extract_input<SocketValueVariant>("Channel"_ustr);
|
||||
SocketValueVariant lows = params.extract_input<SocketValueVariant>("Low"_ustr);
|
||||
SocketValueVariant highs = params.extract_input<SocketValueVariant>("High"_ustr);
|
||||
|
||||
auto sample_fn = std::make_shared<SampleSoundFunction>(
|
||||
*sound, to_fft_size_int(fft_size), to_window_function(window_function));
|
||||
|
||||
SocketValueVariant amplitudes;
|
||||
std::string error_message;
|
||||
if (!execute_multi_function_on_value_variant(std::move(sample_fn),
|
||||
{×, &all_channels, &channels, &lows, &highs},
|
||||
{&litudes},
|
||||
params.user_data(),
|
||||
error_message))
|
||||
{
|
||||
params.set_default_remaining_outputs();
|
||||
params.error_message_add(NodeWarningType::Error, std::move(error_message));
|
||||
return;
|
||||
}
|
||||
|
||||
params.set_output("Amplitude"_ustr, std::move(amplitudes));
|
||||
}
|
||||
|
||||
static void node_register()
|
||||
{
|
||||
static bke::bNodeType ntype;
|
||||
geo_node_type_base(&ntype, "GeometryNodeSampleSoundFrequencies"_ustr);
|
||||
ntype.ui_name = "Sample Sound Frequencies";
|
||||
ntype.ui_description =
|
||||
"Retrieve the amplitude from a sound data-block of a frequency range at a given time";
|
||||
ntype.nclass = NODE_CLASS_CONVERTER;
|
||||
ntype.declare = node_declare;
|
||||
ntype.geometry_node_execute = node_geo_exec;
|
||||
bke::node_type_size(ntype, 180, 100, NODE_DEFAULT_MAX_WIDTH);
|
||||
bke::node_register_type(ntype);
|
||||
}
|
||||
NOD_REGISTER_NODE(node_register)
|
||||
|
||||
} // namespace blender::nodes::node_geo_sample_sound_frequencies_cc
|
||||
Loading…
Add table
Add a link
Reference in a new issue