From 50551d987a8565c7300c5745506e048212fe9a64 Mon Sep 17 00:00:00 2001 From: David Sansome Date: Sun, 21 Sep 2014 19:35:21 +1000 Subject: [PATCH] Rewrite moodbar stuff for gstreamer-1.0: - Rewrite gstspectrum (1.0) to use FFTW (2x faster) and emit raw magnitude values (not log scaled). - Rewrite the moodbar generation code to be somewhat understandable, and do it in Clementine instead of gstreamer. --- CMakeLists.txt | 10 +- gst/moodbar/01-fftw-plan-mutex.patch | 31 -- gst/moodbar/CMakeLists.txt | 8 +- gst/moodbar/gstfastspectrum.cpp | 535 +++++++++++++++++++ gst/moodbar/gstfastspectrum.h | 93 ++++ gst/moodbar/gstfftwspectrum.c | 738 --------------------------- gst/moodbar/gstfftwspectrum.h | 72 --- gst/moodbar/gstmoodbar.c | 679 ------------------------ gst/moodbar/gstmoodbar.h | 63 --- gst/moodbar/plugin.cpp | 48 ++ gst/moodbar/plugin.h | 25 + gst/moodbar/spectrum.c | 71 --- gst/moodbar/spectrum.h | 67 --- src/CMakeLists.txt | 1 + src/engines/gstengine.cpp | 4 +- src/moodbar/moodbarbuilder.cpp | 191 +++++++ src/moodbar/moodbarbuilder.h | 50 ++ src/moodbar/moodbarpipeline.cpp | 79 ++- src/moodbar/moodbarpipeline.h | 9 +- 19 files changed, 1002 insertions(+), 1772 deletions(-) delete mode 100644 gst/moodbar/01-fftw-plan-mutex.patch create mode 100644 gst/moodbar/gstfastspectrum.cpp create mode 100644 gst/moodbar/gstfastspectrum.h delete mode 100644 gst/moodbar/gstfftwspectrum.c delete mode 100644 gst/moodbar/gstfftwspectrum.h delete mode 100644 gst/moodbar/gstmoodbar.c delete mode 100644 gst/moodbar/gstmoodbar.h create mode 100644 gst/moodbar/plugin.cpp create mode 100644 gst/moodbar/plugin.h delete mode 100644 gst/moodbar/spectrum.c delete mode 100644 gst/moodbar/spectrum.h create mode 100644 src/moodbar/moodbarbuilder.cpp create mode 100644 src/moodbar/moodbarbuilder.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bbfb4310..a02a0b474 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,11 +53,12 @@ pkg_check_modules(CHROMAPRINT libchromaprint) pkg_check_modules(GIO gio-2.0) pkg_check_modules(GLIB glib-2.0) pkg_check_modules(GOBJECT gobject-2.0) -pkg_check_modules(GSTREAMER gstreamer-1.0) -pkg_check_modules(GSTREAMER_APP gstreamer-app-1.0) -pkg_check_modules(GSTREAMER_BASE gstreamer-base-1.0) +pkg_check_modules(GSTREAMER REQUIRED gstreamer-1.0) +pkg_check_modules(GSTREAMER_APP REQUIRED gstreamer-app-1.0) +pkg_check_modules(GSTREAMER_AUDIO REQUIRED gstreamer-audio-1.0) +pkg_check_modules(GSTREAMER_BASE REQUIRED gstreamer-base-1.0) #pkg_check_modules(GSTREAMER_CDDA gstreamer-cdda-0.10) -pkg_check_modules(GSTREAMER_TAG gstreamer-tag-1.0) +pkg_check_modules(GSTREAMER_TAG REQUIRED gstreamer-tag-1.0) pkg_check_modules(INDICATEQT indicate-qt) pkg_check_modules(LIBGPOD libgpod-1.0>=0.7.92) pkg_check_modules(LIBMTP libmtp>=1.0) @@ -144,6 +145,7 @@ include_directories(${TAGLIB_INCLUDE_DIRS}) include_directories(${QJSON_INCLUDE_DIRS}) include_directories(${GSTREAMER_INCLUDE_DIRS}) include_directories(${GSTREAMER_APP_INCLUDE_DIRS}) +include_directories(${GSTREAMER_AUDIO_INCLUDE_DIRS}) include_directories(${GSTREAMER_BASE_INCLUDE_DIRS}) include_directories(${GSTREAMER_CDDA_INCLUDE_DIRS}) include_directories(${GSTREAMER_TAG_INCLUDE_DIRS}) diff --git a/gst/moodbar/01-fftw-plan-mutex.patch b/gst/moodbar/01-fftw-plan-mutex.patch deleted file mode 100644 index 8e4862939..000000000 --- a/gst/moodbar/01-fftw-plan-mutex.patch +++ /dev/null @@ -1,31 +0,0 @@ -From 0b62ebc38d1cc0202c6f57c4e096fa0b68a41baf Mon Sep 17 00:00:00 2001 -From: David Sansome -Date: Sun, 27 May 2012 17:00:32 +0100 -Subject: [PATCH] Protect calls to fftwf_plan_dft_r2c_1d with a mutex - ---- - plugin/gstfftwspectrum.c | 4 ++++ - 1 files changed, 4 insertions(+), 0 deletions(-) - -diff --git a/plugin/gstfftwspectrum.c b/plugin/gstfftwspectrum.c -index 147e606..f6e2427 100644 ---- a/plugin/gstfftwspectrum.c -+++ b/plugin/gstfftwspectrum.c -@@ -302,10 +302,14 @@ alloc_fftw_data (GstFFTWSpectrum *conv) - * outputs are the hermetian conjugates). This should be optimal for - * implementing filters. - */ -+ -+ static GStaticMutex mutex = G_STATIC_MUTEX_INIT; -+ g_static_mutex_lock(&mutex); - conv->fftw_plan - = fftwf_plan_dft_r2c_1d(conv->size, conv->fftw_in, - (fftwf_complex *) conv->fftw_out, - conv->hi_q ? FFTW_MEASURE : FFTW_ESTIMATE); -+ g_static_mutex_unlock(&mutex); - } - - --- -1.7.5.4 - diff --git a/gst/moodbar/CMakeLists.txt b/gst/moodbar/CMakeLists.txt index dc3934b7b..44c136253 100644 --- a/gst/moodbar/CMakeLists.txt +++ b/gst/moodbar/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 2.6) set(CMAKE_C_FLAGS "-Wall") -set(CMAKE_CXX_FLAGS "-Woverloaded-virtual -Wall") +set(CMAKE_CXX_FLAGS "-Woverloaded-virtual -Wall --std=c++0x") include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) @@ -11,9 +11,8 @@ include_directories(${GSTREAMER_INCLUDE_DIRS}) include_directories(${FFTW3_INCLUDE_DIR}) set(SOURCES - gstfftwspectrum.c - gstmoodbar.c - spectrum.c + gstfastspectrum.cpp + plugin.cpp ) add_library(gstmoodbar STATIC @@ -24,6 +23,7 @@ target_link_libraries(gstmoodbar ${GOBJECT_LIBRARIES} ${GLIB_LIBRARIES} ${GSTREAMER_LIBRARIES} + ${GSTREAMER_AUDIO_LIBRARIES} ${GSTREAMER_BASE_LIBRARIES} ${FFTW3_FFTW_LIBRARY} ) diff --git a/gst/moodbar/gstfastspectrum.cpp b/gst/moodbar/gstfastspectrum.cpp new file mode 100644 index 000000000..2b57ccacc --- /dev/null +++ b/gst/moodbar/gstfastspectrum.cpp @@ -0,0 +1,535 @@ +/* GStreamer + * Copyright (C) <1999> Erik Walthinsen + * <2006,2011> Stefan Kost + * <2007-2009> Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include +#include +#include "gstfastspectrum.h" + +GST_DEBUG_CATEGORY_STATIC (gst_fastspectrum_debug); +#define GST_CAT_DEFAULT gst_fastspectrum_debug + +/* elementfactory information */ +#if G_BYTE_ORDER == G_LITTLE_ENDIAN +# define FORMATS "{ S16LE, S24LE, S32LE, F32LE, F64LE }" +#else +# define FORMATS "{ S16BE, S24BE, S32BE, F32BE, F64BE }" +#endif + +#define ALLOWED_CAPS \ + GST_AUDIO_CAPS_MAKE (FORMATS) ", " \ + "layout = (string) interleaved, " \ + "channels = 1" + +/* Spectrum properties */ +#define DEFAULT_INTERVAL (GST_SECOND / 10) +#define DEFAULT_BANDS 128 + +enum { + PROP_0, + PROP_INTERVAL, + PROP_BANDS +}; + +#define gst_fastspectrum_parent_class parent_class +G_DEFINE_TYPE (GstFastSpectrum, gst_fastspectrum, GST_TYPE_AUDIO_FILTER); + +static void gst_fastspectrum_finalize (GObject * object); +static void gst_fastspectrum_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_fastspectrum_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); +static gboolean gst_fastspectrum_start (GstBaseTransform * trans); +static gboolean gst_fastspectrum_stop (GstBaseTransform * trans); +static GstFlowReturn gst_fastspectrum_transform_ip (GstBaseTransform * trans, + GstBuffer * in); +static gboolean gst_fastspectrum_setup (GstAudioFilter * base, + const GstAudioInfo * info); + +static void +gst_fastspectrum_class_init (GstFastSpectrumClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstBaseTransformClass *trans_class = GST_BASE_TRANSFORM_CLASS (klass); + GstAudioFilterClass *filter_class = GST_AUDIO_FILTER_CLASS (klass); + GstCaps *caps; + + gobject_class->set_property = gst_fastspectrum_set_property; + gobject_class->get_property = gst_fastspectrum_get_property; + gobject_class->finalize = gst_fastspectrum_finalize; + + trans_class->start = GST_DEBUG_FUNCPTR (gst_fastspectrum_start); + trans_class->stop = GST_DEBUG_FUNCPTR (gst_fastspectrum_stop); + trans_class->transform_ip = GST_DEBUG_FUNCPTR (gst_fastspectrum_transform_ip); + trans_class->passthrough_on_same_caps = TRUE; + + filter_class->setup = GST_DEBUG_FUNCPTR (gst_fastspectrum_setup); + + g_object_class_install_property (gobject_class, PROP_INTERVAL, + g_param_spec_uint64 ("interval", "Interval", + "Interval of time between message posts (in nanoseconds)", + 1, G_MAXUINT64, DEFAULT_INTERVAL, + GParamFlags(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_BANDS, + g_param_spec_uint ("bands", "Bands", "Number of frequency bands", + 0, G_MAXUINT, DEFAULT_BANDS, + GParamFlags(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + GST_DEBUG_CATEGORY_INIT (gst_fastspectrum_debug, "spectrum", 0, + "audio spectrum analyser element"); + + gst_element_class_set_static_metadata (element_class, "Spectrum analyzer", + "Filter/Analyzer/Audio", + "Run an FFT on the audio signal, output spectrum data", + "Erik Walthinsen , " + "Stefan Kost , " + "Sebastian Dröge "); + + caps = gst_caps_from_string (ALLOWED_CAPS); + gst_audio_filter_class_add_pad_templates (filter_class, caps); + gst_caps_unref (caps); +} + +static void +gst_fastspectrum_init (GstFastSpectrum * spectrum) +{ + spectrum->interval = DEFAULT_INTERVAL; + spectrum->bands = DEFAULT_BANDS; + + spectrum->channel_data_initialised = false; + + g_mutex_init (&spectrum->lock); +} + +static void +gst_fastspectrum_alloc_channel_data (GstFastSpectrum * spectrum) +{ + guint bands = spectrum->bands; + guint nfft = 2 * bands - 2; + + spectrum->input_ring_buffer = new double[nfft]; + spectrum->fft_input = reinterpret_cast( + fftw_malloc(sizeof(double) * nfft)); + spectrum->fft_output =reinterpret_cast( + fftw_malloc(sizeof(fftw_complex) * (nfft/2+1))); + + spectrum->spect_magnitude = new double[bands]; + spectrum->plan = fftw_plan_dft_r2c_1d( + nfft, + spectrum->fft_input, + spectrum->fft_output, + FFTW_ESTIMATE); + spectrum->channel_data_initialised = true; +} + +static void +gst_fastspectrum_free_channel_data (GstFastSpectrum * spectrum) +{ + if (spectrum->channel_data_initialised) { + fftw_destroy_plan(spectrum->plan); + fftw_free(spectrum->fft_input); + fftw_free(spectrum->fft_output); + delete[] spectrum->input_ring_buffer; + delete[] spectrum->spect_magnitude; + + spectrum->channel_data_initialised = false; + } +} + +static void +gst_fastspectrum_flush (GstFastSpectrum * spectrum) +{ + spectrum->num_frames = 0; + spectrum->num_fft = 0; + + spectrum->accumulated_error = 0; +} + +static void +gst_fastspectrum_reset_state (GstFastSpectrum * spectrum) +{ + GST_DEBUG_OBJECT (spectrum, "resetting state"); + + gst_fastspectrum_free_channel_data (spectrum); + gst_fastspectrum_flush (spectrum); +} + +static void +gst_fastspectrum_finalize (GObject * object) +{ + GstFastSpectrum *spectrum = GST_FASTSPECTRUM (object); + + gst_fastspectrum_reset_state (spectrum); + g_mutex_clear (&spectrum->lock); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_fastspectrum_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstFastSpectrum *filter = GST_FASTSPECTRUM (object); + + switch (prop_id) { + case PROP_INTERVAL:{ + guint64 interval = g_value_get_uint64 (value); + g_mutex_lock (&filter->lock); + if (filter->interval != interval) { + filter->interval = interval; + gst_fastspectrum_reset_state (filter); + } + g_mutex_unlock (&filter->lock); + break; + } + case PROP_BANDS:{ + guint bands = g_value_get_uint (value); + g_mutex_lock (&filter->lock); + if (filter->bands != bands) { + filter->bands = bands; + gst_fastspectrum_reset_state (filter); + } + g_mutex_unlock (&filter->lock); + break; + } + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_fastspectrum_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstFastSpectrum *filter = GST_FASTSPECTRUM (object); + + switch (prop_id) { + case PROP_INTERVAL: + g_value_set_uint64 (value, filter->interval); + break; + case PROP_BANDS: + g_value_set_uint (value, filter->bands); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +gst_fastspectrum_start (GstBaseTransform * trans) +{ + GstFastSpectrum *spectrum = GST_FASTSPECTRUM (trans); + + gst_fastspectrum_reset_state (spectrum); + + return TRUE; +} + +static gboolean +gst_fastspectrum_stop (GstBaseTransform * trans) +{ + GstFastSpectrum *spectrum = GST_FASTSPECTRUM (trans); + + gst_fastspectrum_reset_state (spectrum); + + return TRUE; +} + +/* mixing data readers */ + +static void +input_data_mixed_float(const guint8* _in, double* out, guint len, + double max_value, guint op, guint nfft) +{ + guint j, ip = 0; + gfloat *in = (gfloat *) _in; + + for (j = 0; j < len; j++) { + out[op] = in[ip++]; + op = (op + 1) % nfft; + } +} + +static void +input_data_mixed_double (const guint8 * _in, double* out, guint len, + double max_value, guint op, guint nfft) +{ + guint j, ip = 0; + gdouble *in = (gdouble *) _in; + + for (j = 0; j < len; j++) { + out[op] = in[ip++]; + op = (op + 1) % nfft; + } +} + +static void +input_data_mixed_int32_max (const guint8 * _in, double* out, guint len, + double max_value, guint op, guint nfft) +{ + guint j, ip = 0; + gint32 *in = (gint32 *) _in; + + for (j = 0; j < len; j++) { + out[op] = in[ip++] / max_value; + op = (op + 1) % nfft; + } +} + +static void +input_data_mixed_int24_max (const guint8 * _in, double* out, guint len, + double max_value, guint op, guint nfft) +{ + guint j; + + for (j = 0; j < len; j++) { +#if G_BYTE_ORDER == G_BIG_ENDIAN + gint32 value = GST_READ_UINT24_BE (_in); +#else + gint32 value = GST_READ_UINT24_LE (_in); +#endif + if (value & 0x00800000) + value |= 0xff000000; + + out[op] = value / max_value; + op = (op + 1) % nfft; + _in += 3; + } +} + +static void +input_data_mixed_int16_max (const guint8 * _in, double * out, guint len, + double max_value, guint op, guint nfft) +{ + guint j, ip = 0; + gint16 *in = (gint16 *) _in; + + for (j = 0; j < len; j++) { + out[op] = in[ip++] / max_value; + op = (op + 1) % nfft; + } +} + +static gboolean +gst_fastspectrum_setup (GstAudioFilter * base, const GstAudioInfo * info) +{ + GstFastSpectrum *spectrum = GST_FASTSPECTRUM (base); + GstFastSpectrumInputData input_data = NULL; + + g_mutex_lock (&spectrum->lock); + switch (GST_AUDIO_INFO_FORMAT (info)) { + case GST_AUDIO_FORMAT_S16: + input_data = input_data_mixed_int16_max; + break; + case GST_AUDIO_FORMAT_S24: + input_data = input_data_mixed_int24_max; + break; + case GST_AUDIO_FORMAT_S32: + input_data = input_data_mixed_int32_max; + break; + case GST_AUDIO_FORMAT_F32: + input_data = input_data_mixed_float; + break; + case GST_AUDIO_FORMAT_F64: + input_data = input_data_mixed_double; + break; + default: + g_assert_not_reached (); + break; + } + spectrum->input_data = input_data; + + gst_fastspectrum_reset_state (spectrum); + g_mutex_unlock (&spectrum->lock); + + return TRUE; +} + +static void +gst_fastspectrum_run_fft (GstFastSpectrum * spectrum, guint input_pos) +{ + guint i; + guint bands = spectrum->bands; + guint nfft = 2 * bands - 2; + + for (i = 0; i < nfft; i++) + spectrum->fft_input[i] = + spectrum->input_ring_buffer[(input_pos + i) % nfft]; + + fftw_execute(spectrum->plan); + + gdouble val; + /* Calculate magnitude in db */ + for (i = 0; i < bands; i++) { + val = spectrum->fft_output[i][0] * spectrum->fft_output[i][0]; + val += spectrum->fft_output[i][1] * spectrum->fft_output[i][1]; + val /= nfft * nfft; + spectrum->spect_magnitude[i] += val; + } +} + +static GstFlowReturn +gst_fastspectrum_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) +{ + GstFastSpectrum *spectrum = GST_FASTSPECTRUM (trans); + guint rate = GST_AUDIO_FILTER_RATE (spectrum); + guint bps = GST_AUDIO_FILTER_BPS (spectrum); + guint bpf = GST_AUDIO_FILTER_BPF (spectrum); + double max_value = (1UL << ((bps << 3) - 1)) - 1; + guint bands = spectrum->bands; + guint nfft = 2 * bands - 2; + guint input_pos; + GstMapInfo map; + const guint8 *data; + gsize size; + guint fft_todo, msg_todo, block_size; + gboolean have_full_interval; + GstFastSpectrumInputData input_data; + + g_mutex_lock (&spectrum->lock); + gst_buffer_map (buffer, &map, GST_MAP_READ); + data = map.data; + size = map.size; + + GST_LOG_OBJECT (spectrum, "input size: %" G_GSIZE_FORMAT " bytes", size); + + if (GST_BUFFER_IS_DISCONT (buffer)) { + GST_DEBUG_OBJECT (spectrum, "Discontinuity detected -- flushing"); + gst_fastspectrum_flush (spectrum); + } + + /* If we don't have a FFT context yet (or it was reset due to parameter + * changes) get one and allocate memory for everything + */ + if (!spectrum->channel_data_initialised) { + GST_DEBUG_OBJECT (spectrum, "allocating for bands %u", bands); + + gst_fastspectrum_alloc_channel_data (spectrum); + + /* number of sample frames we process before posting a message + * interval is in ns */ + spectrum->frames_per_interval = + gst_util_uint64_scale (spectrum->interval, rate, GST_SECOND); + spectrum->frames_todo = spectrum->frames_per_interval; + /* rounding error for frames_per_interval in ns, + * aggregated it in accumulated_error */ + spectrum->error_per_interval = (spectrum->interval * rate) % GST_SECOND; + if (spectrum->frames_per_interval == 0) + spectrum->frames_per_interval = 1; + + GST_INFO_OBJECT (spectrum, "interval %" GST_TIME_FORMAT ", fpi %" + G_GUINT64_FORMAT ", error %" GST_TIME_FORMAT, + GST_TIME_ARGS (spectrum->interval), spectrum->frames_per_interval, + GST_TIME_ARGS (spectrum->error_per_interval)); + + spectrum->input_pos = 0; + + gst_fastspectrum_flush (spectrum); + } + + if (spectrum->num_frames == 0) + spectrum->message_ts = GST_BUFFER_TIMESTAMP (buffer); + + input_pos = spectrum->input_pos; + input_data = spectrum->input_data; + + while (size >= bpf) { + /* run input_data for a chunk of data */ + fft_todo = nfft - (spectrum->num_frames % nfft); + msg_todo = spectrum->frames_todo - spectrum->num_frames; + GST_LOG_OBJECT (spectrum, + "message frames todo: %u, fft frames todo: %u, input frames %" + G_GSIZE_FORMAT, msg_todo, fft_todo, (size / bpf)); + block_size = msg_todo; + if (block_size > (size / bpf)) + block_size = (size / bpf); + if (block_size > fft_todo) + block_size = fft_todo; + + /* Move the current frames into our ringbuffers */ + input_data(data, spectrum->input_ring_buffer, block_size, max_value, input_pos, nfft); + + data += block_size * bpf; + size -= block_size * bpf; + input_pos = (input_pos + block_size) % nfft; + spectrum->num_frames += block_size; + + have_full_interval = (spectrum->num_frames == spectrum->frames_todo); + + GST_LOG_OBJECT (spectrum, + "size: %" G_GSIZE_FORMAT ", do-fft = %d, do-message = %d", size, + (spectrum->num_frames % nfft == 0), have_full_interval); + + /* If we have enough frames for an FFT or we have all frames required for + * the interval and we haven't run a FFT, then run an FFT */ + if ((spectrum->num_frames % nfft == 0) || + (have_full_interval && !spectrum->num_fft)) { + gst_fastspectrum_run_fft (spectrum, input_pos); + spectrum->num_fft++; + } + + /* Do we have the FFTs for one interval? */ + if (have_full_interval) { + GST_DEBUG_OBJECT (spectrum, "nfft: %u frames: %" G_GUINT64_FORMAT + " fpi: %" G_GUINT64_FORMAT " error: %" GST_TIME_FORMAT, nfft, + spectrum->num_frames, spectrum->frames_per_interval, + GST_TIME_ARGS (spectrum->accumulated_error)); + + spectrum->frames_todo = spectrum->frames_per_interval; + if (spectrum->accumulated_error >= GST_SECOND) { + spectrum->accumulated_error -= GST_SECOND; + spectrum->frames_todo++; + } + spectrum->accumulated_error += spectrum->error_per_interval; + + if (spectrum->output_callback) { + // Calculate average + for (uint i = 0; i < spectrum->bands; i++) { + spectrum->spect_magnitude[i] /= spectrum->num_fft; + } + + spectrum->output_callback(spectrum->spect_magnitude, spectrum->bands); + + // Reset spectrum accumulators + memset(spectrum->spect_magnitude, 0, spectrum->bands * sizeof(double)); + } + + if (GST_CLOCK_TIME_IS_VALID (spectrum->message_ts)) + spectrum->message_ts += + gst_util_uint64_scale (spectrum->num_frames, GST_SECOND, rate); + + spectrum->num_frames = 0; + spectrum->num_fft = 0; + } + } + + spectrum->input_pos = input_pos; + + gst_buffer_unmap (buffer, &map); + g_mutex_unlock (&spectrum->lock); + + g_assert (size == 0); + + return GST_FLOW_OK; +} diff --git a/gst/moodbar/gstfastspectrum.h b/gst/moodbar/gstfastspectrum.h new file mode 100644 index 000000000..6147f3eb9 --- /dev/null +++ b/gst/moodbar/gstfastspectrum.h @@ -0,0 +1,93 @@ +/* GStreamer + * Copyright (C) <1999> Erik Walthinsen + * Copyright (C) <2009> Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +// Adapted from gstspectrum for Clementine with the following changes: +// - Uses fftw instead of kiss fft (2x faster). +// - Hardcoded to 1 channel (use an audioconvert element to do the work +// instead, simplifies this code a lot). +// - Send output via a callback instead of GST messages (less overhead). +// - Removed all properties except interval and band. + + +#ifndef GST_MOODBAR_FASTSPECTRUM_H_ +#define GST_MOODBAR_FASTSPECTRUM_H_ + +#include + +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_FASTSPECTRUM (gst_fastspectrum_get_type()) +#define GST_FASTSPECTRUM(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_FASTSPECTRUM,GstFastSpectrum)) +#define GST_IS_FASTSPECTRUM(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_FASTSPECTRUM)) +#define GST_FASTSPECTRUM_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_FASTSPECTRUM,GstFastSpectrumClass)) +#define GST_IS_FASTSPECTRUM_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_FASTSPECTRUM)) + +using GstFastSpectrumInputData = void(*)(const guint8* in, double* out, + guint len, double max_value, guint op, guint nfft); + +using OutputCallback = std::function; + +struct GstFastSpectrum { + GstAudioFilter parent; + + /* properties */ + guint64 interval; /* how many nanoseconds between emits */ + guint64 frames_per_interval; /* how many frames per interval */ + guint64 frames_todo; + guint bands; /* number of spectrum bands */ + gboolean multi_channel; /* send separate channel results */ + + guint64 num_frames; /* frame count (1 sample per channel) + * since last emit */ + guint64 num_fft; /* number of FFTs since last emit */ + GstClockTime message_ts; /* starttime for next message */ + + /* */ + bool channel_data_initialised; + double* input_ring_buffer; + double* fft_input; + fftw_complex* fft_output; + double* spect_magnitude; + fftw_plan plan; + + guint input_pos; + guint64 error_per_interval; + guint64 accumulated_error; + + GMutex lock; + + GstFastSpectrumInputData input_data; + + OutputCallback output_callback; +}; + +struct GstFastSpectrumClass { + GstAudioFilterClass parent_class; +}; + +GType gst_fastspectrum_get_type (void); + +G_END_DECLS + +#endif // GST_MOODBAR_FASTSPECTRUM_H_ diff --git a/gst/moodbar/gstfftwspectrum.c b/gst/moodbar/gstfftwspectrum.c deleted file mode 100644 index bda6291e5..000000000 --- a/gst/moodbar/gstfftwspectrum.c +++ /dev/null @@ -1,738 +0,0 @@ -/* GStreamer FFTW-based signal-to-spectrum converter - * Copyright (C) 2006 Joseph Rabinoff - */ - -/*************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ - -/** - * SECTION:element-fftwspectrum - * - * - * Example launch line - * - * - * gst-launch audiotestsrc ! audioconvert ! fftwspectrum ! fftwunspectrum ! audioconvert ! alsasink - * - * - * - */ - -/* This is a simple plugin to take an audio signal and return its - * Fourier transform, using fftw3. It takes a specified number N of - * samples and returns the first N/2+1 (complex) Fourier transform - * values (the other half of the values being the complex conjugates - * of the first). The modulus of these values correspond to the - * strength of the signal in their various bands, and the phase gives - * information about the phase of the signal. The step by which the - * transform increments is also variable, so it can return redundant - * data (to reduce artifacts when converting back into a signal). - */ - -#ifdef HAVE_CONFIG_H -# include -#endif - -#include -#include -#include -#include - -#include "gstfftwspectrum.h" -#include "spectrum.h" - -GST_DEBUG_CATEGORY (gst_fftwspectrum_debug); -#define GST_CAT_DEFAULT gst_fftwspectrum_debug - -/* Filter signals and args */ -enum -{ - /* FILL ME */ - LAST_SIGNAL -}; - -/* The size and step arguments are actually only default values - * used to fixate the size and step properties of the source cap. - */ -enum -{ - ARG_0, - ARG_DEF_SIZE, - ARG_DEF_STEP, - ARG_HIQUALITY -}; - -#define DEF_SIZE_DEFAULT 1024 -#define DEF_STEP_DEFAULT 512 -#define HIQUALITY_DEFAULT TRUE - -static GstStaticPadTemplate sink_factory - = GST_STATIC_PAD_TEMPLATE ("sink", - GST_PAD_SINK, - GST_PAD_ALWAYS, - GST_STATIC_CAPS - ( SPECTRUM_SIGNAL_CAPS ) - ); - -/* See spectrum.h for a definition of the frequency caps */ -static GstStaticPadTemplate src_factory - = GST_STATIC_PAD_TEMPLATE ("src", - GST_PAD_SRC, - GST_PAD_ALWAYS, - GST_STATIC_CAPS - ( SPECTRUM_FREQ_CAPS ) - ); - -G_DEFINE_TYPE(GstFFTWSpectrum, gst_fftwspectrum, GST_TYPE_ELEMENT); - -static void gst_fftwspectrum_set_property (GObject *object, guint prop_id, - const GValue *value, GParamSpec *pspec); -static void gst_fftwspectrum_get_property (GObject *object, guint prop_id, - GValue *value, GParamSpec *pspec); - -static gboolean gst_fftwspectrum_sink_event( - GstPad* pad, GstObject* parent, GstEvent* event); -static gboolean gst_fftwspectrum_src_event( - GstPad* pad, GstObject* parent, GstEvent* event); -static gboolean gst_fftwspectrum_src_query( - GstPad* pad, GstObject* parent, GstQuery* query); -static gboolean gst_fftwspectrum_sink_query( - GstPad* pad, GstObject* parent, GstQuery* query); -static gboolean gst_fftwspectrum_set_sink_caps (GstPad *pad, GstCaps *caps); -static gboolean gst_fftwspectrum_set_src_caps (GstPad *pad, GstCaps *caps); -static void gst_fftwspectrum_fixatecaps (GstPad *pad, GstCaps *caps); -static GstCaps *gst_fftwspectrum_getcaps (GstPad *pad); - -static GstFlowReturn gst_fftwspectrum_chain( - GstPad *pad, GstObject* object, GstBuffer *buf); -static GstStateChangeReturn gst_fftwspectrum_change_state (GstElement *element, - GstStateChange transition); - - -#define OUTPUT_SIZE(conv) (((conv)->size/2+1)*sizeof(fftw_complex)) - - -/***************************************************************/ -/* GObject boilerplate stuff */ -/***************************************************************/ - - -/* initialize the plugin's class */ -static void -gst_fftwspectrum_class_init (GstFFTWSpectrumClass * klass) -{ - GObjectClass *gobject_class; - GstElementClass *gstelement_class; - - gobject_class = (GObjectClass *) klass; - gstelement_class = (GstElementClass *) klass; - - gobject_class->set_property = gst_fftwspectrum_set_property; - gobject_class->get_property = gst_fftwspectrum_get_property; - - g_object_class_install_property (gobject_class, ARG_DEF_SIZE, - g_param_spec_int ("def-size", "Default Size", - "Apply a Fourier transform to this many samples at a time (default value)", - 1, G_MAXINT32, DEF_SIZE_DEFAULT, G_PARAM_READWRITE)); - - g_object_class_install_property (gobject_class, ARG_DEF_STEP, - g_param_spec_int ("def-step", "Default Step", - "Advance the stream this many samples each time (default value)", - 1, G_MAXINT32, DEF_STEP_DEFAULT, G_PARAM_READWRITE)); - - g_object_class_install_property (gobject_class, ARG_HIQUALITY, - g_param_spec_boolean ("hiquality", "High Quality", - "Use a more time-consuming, higher quality algorithm chooser", - HIQUALITY_DEFAULT, G_PARAM_READWRITE)); - - gstelement_class->change_state - = GST_DEBUG_FUNCPTR (gst_fftwspectrum_change_state); - - gst_element_class_add_pad_template (GST_ELEMENT_CLASS(klass), - gst_static_pad_template_get (&src_factory)); - gst_element_class_add_pad_template (GST_ELEMENT_CLASS(klass), - gst_static_pad_template_get (&sink_factory)); - - gst_element_class_set_static_metadata( - GST_ELEMENT_CLASS(klass), - "FFTW-based Fourier transform", - "Filter/Converter/Spectrum", - "Convert a raw audio stream into a frequency spectrum", - "Joe Rabinoff "); - - g_mutex_init(&klass->mutex); -} - -/* initialize the new element - * instantiate pads and add them to element - * set functions - * initialize structure - */ -static void -gst_fftwspectrum_init (GstFFTWSpectrum * conv) -{ - GstElementClass* klass = - G_TYPE_INSTANCE_GET_CLASS(conv, GST_ELEMENT_TYPE, GstElementClass); - - conv->sinkpad = - gst_pad_new_from_template - (gst_element_class_get_pad_template (klass, "sink"), "sink"); - gst_pad_set_event_function(conv->sinkpad, gst_fftwspectrum_sink_event); - gst_pad_set_query_function(conv->srcpad, gst_fftwspectrum_src_query); - - gst_pad_set_chain_function (conv->sinkpad, - GST_DEBUG_FUNCPTR (gst_fftwspectrum_chain)); - - conv->srcpad = - gst_pad_new_from_template - (gst_element_class_get_pad_template (klass, "src"), "src"); - gst_pad_set_event_function(conv->srcpad, gst_fftwspectrum_src_event); - gst_pad_set_query_function(conv->sinkpad, gst_fftwspectrum_sink_query); - - //gst_pad_set_fixatecaps_function (conv->srcpad, - // GST_DEBUG_FUNCPTR (gst_fftwspectrum_fixatecaps)); - - - gst_element_add_pad (GST_ELEMENT (conv), conv->sinkpad); - gst_element_add_pad (GST_ELEMENT (conv), conv->srcpad); - - /* These are set once the (source) capabilities are determined */ - conv->rate = 0; - conv->size = 0; - conv->step = 0; - - /* These are set when we change to READY */ - conv->fftw_in = NULL; - conv->fftw_out = NULL; - conv->fftw_plan = NULL; - - /* These are set when we start receiving data */ - conv->samples = NULL; - conv->numsamples = 0; - conv->timestamp = 0; - conv->offset = 0; - - /* Properties */ - conv->def_size = DEF_SIZE_DEFAULT; - conv->def_step = DEF_STEP_DEFAULT; - conv->hi_q = HIQUALITY_DEFAULT; - - conv->mutex = &gclass->mutex; -} - -static gboolean -gst_fftwspectrum_sink_event(GstPad* pad, GstObject* parent, GstEvent* event) -{ - GstFFTWSpectrum* conv = GST_FFTWSPECTRUM(parent); - switch (GST_EVENT_TYPE(event)) { - case GST_EVENT_CAPS: { - GstCaps* caps = NULL; - gst_event_parse_caps(event, &caps); - gst_fftwspectrum_set_sink_caps(pad, caps); - return gst_pad_push_event(conv->srcpad, event); - } - default: - return gst_pad_event_default(pad, parent, event); - } -} - -static gboolean -gst_fftwspectrum_src_event(GstPad* pad, GstObject* parent, GstEvent* event) -{ - switch (GST_EVENT_TYPE(event)) { - case GST_EVENT_CAPS: { - GstCaps* caps = NULL; - gst_event_parse_caps(event, &caps); - gst_fftwspectrum_set_src_caps(pad, caps); - } - // FALLTHROUGH - default: - return gst_pad_event_default(pad, parent, event); - } -} - -static gboolean -gst_fftwspectrum_src_query(GstPad* pad, GstObject* parent, GstQuery* query) -{ - switch (GST_QUERY_TYPE(query)) { - case GST_QUERY_CAPS: { - GstCaps* caps = gst_fftwspectrum_getcaps(pad); - gst_pad_set_caps(pad, caps); - } - // FALLTHROUGH - default: - return gst_pad_query_default(pad, parent, query); - } -} - -static gboolean -gst_fftwspectrum_sink_query(GstPad* pad, GstObject* parent, GstQuery* query) -{ - switch (GST_QUERY_TYPE(query)) { - case GST_QUERY_CAPS: { - GstCaps* caps = gst_fftwspectrum_getcaps(pad); - gst_pad_set_caps(pad, caps); - } - // FALLTHROUGH - default: - return gst_pad_query_default(pad, parent, query); - } -} - -static void -gst_fftwspectrum_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec) -{ - GstFFTWSpectrum *conv = GST_FFTWSPECTRUM (object); - - switch (prop_id) - { - case ARG_DEF_SIZE: - conv->def_size = g_value_get_int (value); - break; - case ARG_DEF_STEP: - conv->def_step = g_value_get_int (value); - break; - case ARG_HIQUALITY: - conv->hi_q = g_value_get_boolean (value); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - -static void -gst_fftwspectrum_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec) -{ - GstFFTWSpectrum *conv = GST_FFTWSPECTRUM (object); - - switch (prop_id) - { - case ARG_DEF_SIZE: - g_value_set_int (value, conv->def_size); - break; - case ARG_DEF_STEP: - g_value_set_int (value, conv->def_step); - break; - case ARG_HIQUALITY: - g_value_set_boolean (value, conv->hi_q); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - - -/* Allocate and deallocate fftw state data */ -static void -free_fftw_data (GstFFTWSpectrum *conv) -{ - if(conv->fftw_plan != NULL) - fftw_destroy_plan (conv->fftw_plan); - if(conv->fftw_in != NULL) - fftw_free (conv->fftw_in); - if(conv->fftw_out != NULL) - fftw_free (conv->fftw_out); - - conv->fftw_in = NULL; - conv->fftw_out = NULL; - conv->fftw_plan = NULL; -} - -static void -alloc_fftw_data (GstFFTWSpectrum *conv) -{ - free_fftw_data (conv); - - GST_DEBUG ("Allocating data for size = %d and step = %d", - conv->size, conv->step); - - conv->fftw_in = (double *) fftw_malloc (sizeof(double) * conv->size); - conv->fftw_out = (double *) fftw_malloc (OUTPUT_SIZE (conv)); - - /* We use the simplest real-to-complex algorithm, which takes n real - * inputs and returns floor(n/2) + 1 complex outputs (the other n/2 - * outputs are the hermetian conjugates). This should be optimal for - * implementing filters. - */ - - g_mutex_lock(conv->mutex); - conv->fftw_plan - = fftw_plan_dft_r2c_1d(conv->size, conv->fftw_in, - (fftw_complex *) conv->fftw_out, - conv->hi_q ? FFTW_MEASURE : FFTW_ESTIMATE); - g_mutex_unlock(conv->mutex); -} - - -/***************************************************************/ -/* Capabilities negotiation */ -/***************************************************************/ - -/* The input and output capabilities are only related by the "rate" - * parameter, which is propagated so that an audio signal can be - * reconstructed eventually. This module does no rate conversion. - * - * The way I understand it, there are two times when caps negotiation - * takes place: (1) when a sink pad receives either its first buffer, - * or a buffer with a new caps type, and (2) when a source pad request - * a buffer from something downstream, and the returned allocated - * buffer has different caps from the ones already negotiated. In the - * first case, _set_sink_caps is called, and in the second, _set_src_caps - * is called. - * When (1) occurs, we remember the rate (the only variable parameter - * in the source) and set the source caps. Then _set_src_caps is called. - * In _set_src_caps, we check that the rate hasn't changed, and figure out - * or remember appropriate size and step attributes. If _set_src_caps is - * called from _set_sink_caps, this completes our setting up our internal - * configuration; if it is called from (2), we reconfigure just the source - * part of the internal configuration. - */ - -static gboolean -gst_fftwspectrum_set_sink_caps (GstPad * pad, GstCaps * caps) -{ - GstFFTWSpectrum *conv; - GstCaps *srccaps, *newsrccaps; - GstStructure *newstruct; - gint rate; - gboolean res; - - conv = GST_FFTWSPECTRUM (gst_pad_get_parent (pad)); - - srccaps = gst_pad_get_allowed_caps (conv->srcpad); - newsrccaps = gst_caps_copy_nth (srccaps, 0); - gst_caps_unref (srccaps); - - newstruct = gst_caps_get_structure (caps, 0); - if (!gst_structure_get_int (newstruct, "rate", &rate)) - { - gst_caps_unref (newsrccaps); - gst_object_unref (conv); - return FALSE; - } - - /* Fixate the source caps with the given rate */ - gst_caps_set_simple (newsrccaps, "rate", G_TYPE_INT, rate, NULL); - //gst_pad_fixate_caps (conv->srcpad, newsrccaps); - conv->rate = rate; - res = gst_pad_set_caps (conv->srcpad, newsrccaps); - if (!res) - conv->rate = 0; - - gst_caps_unref (newsrccaps); - gst_object_unref (conv); - - return res; -} - -static gboolean -gst_fftwspectrum_set_src_caps (GstPad * pad, GstCaps * caps) -{ - GstFFTWSpectrum *conv; - gboolean res = FALSE; - GstStructure *newstruct; - gint rate; - - conv = GST_FFTWSPECTRUM (gst_pad_get_parent (pad)); - - newstruct = gst_caps_get_structure (caps, 0); - if (!gst_structure_get_int (newstruct, "rate", &rate)) - goto out; - - /* Assume caps negotiation has already taken place */ - if (rate == conv->rate) - { - gint size, step; - - if (!gst_structure_get_int (newstruct, "size", &size)) - goto out; - if (!gst_structure_get_int (newstruct, "step", &step)) - goto out; - - if (conv->size != size || conv->step != step) - { - conv->size = size; - conv->step = step; - - /* Re-allocate the fftw data */ - if (GST_STATE (GST_ELEMENT (conv)) >= GST_STATE_READY) - alloc_fftw_data (conv); - } - - res = TRUE; - } - - out: - gst_object_unref (conv); - - return res; -} - - -/* The only thing that can constrain the caps is the rate. */ -static GstCaps * -gst_fftwspectrum_getcaps (GstPad *pad) -{ - GstFFTWSpectrum *conv; - GstCaps *tmplcaps; - - conv = GST_FFTWSPECTRUM (gst_pad_get_parent (pad)); - tmplcaps = gst_caps_copy (gst_pad_get_pad_template_caps (pad)); - - if(conv->rate != 0) - { - /* Assumes the template caps are simple */ - gst_caps_set_simple (tmplcaps, "rate", G_TYPE_INT, conv->rate, NULL); - } - - gst_object_unref (conv); - - return tmplcaps; -} - - -/* This is called when the source pad needs to choose its capabilities - * when it has a choice and nobody's forcing its hand. In this case - * we take our hint from the def_size and def_step properties. - */ -static void -gst_fftwspectrum_fixatecaps (GstPad *pad, GstCaps *caps) -{ - GstFFTWSpectrum *conv; - GstStructure *s; - const GValue *val; - - conv = GST_FFTWSPECTRUM (gst_pad_get_parent (pad)); - s = gst_caps_get_structure (caps, 0); - - val = gst_structure_get_value (s, "size"); - if (val == NULL) - gst_caps_set_simple (caps, "size", G_TYPE_INT, conv->def_size, NULL); - else if (G_VALUE_TYPE (val) == GST_TYPE_INT_RANGE) - { - gint sizemin, sizemax; - sizemin = gst_value_get_int_range_min (val); - sizemax = gst_value_get_int_range_max (val); - gst_caps_set_simple (caps, "size", G_TYPE_INT, - CLAMP (conv->def_size, sizemin, sizemax), NULL); - } - /* else it should be already fixed */ - - val = gst_structure_get_value (s, "step"); - if (val == NULL) - gst_caps_set_simple (caps, "step", G_TYPE_INT, conv->def_step, NULL); - else if (G_VALUE_TYPE (val) == GST_TYPE_INT_RANGE) - { - gint stepmin, stepmax; - stepmin = gst_value_get_int_range_min (val); - stepmax = gst_value_get_int_range_max (val); - gst_caps_set_simple (caps, "step", G_TYPE_INT, - CLAMP (conv->def_step, stepmin, stepmax), NULL); - } - /* else it should be already fixed */ - - /* Assume rate is already fixed (if not it'll be fixed by default) */ - - gst_object_unref (conv); -} - - -/***************************************************************/ -/* Actual conversion */ -/***************************************************************/ - - -static GstStateChangeReturn -gst_fftwspectrum_change_state (GstElement * element, - GstStateChange transition) -{ - GstFFTWSpectrum *conv = GST_FFTWSPECTRUM (element); - GstStateChangeReturn res; - - switch (transition) - { - case GST_STATE_CHANGE_NULL_TO_READY: - alloc_fftw_data (conv); - break; - case GST_STATE_CHANGE_READY_TO_PAUSED: - conv->samples = (gdouble *) g_malloc (sizeof(gdouble)); - conv->numsamples = 0; - conv->timestamp = 0; - conv->offset = 0; - break; - case GST_STATE_CHANGE_PAUSED_TO_PLAYING: - break; - default: - break; - } - - res = GST_ELEMENT_CLASS(gst_fftwspectrum_parent_class) - ->change_state(element, transition); - - switch (transition) - { - case GST_STATE_CHANGE_PLAYING_TO_PAUSED: - break; - case GST_STATE_CHANGE_PAUSED_TO_READY: - g_free(conv->samples); - conv->samples = NULL; - conv->numsamples = 0; - conv->timestamp = 0; - conv->offset = 0; - break; - case GST_STATE_CHANGE_READY_TO_NULL: - free_fftw_data (conv); - break; - default: - break; - } - - return res; -} - - -/* Adds the samples contained in buf to the end of conv->samples, - * updating conv->numsamples. - */ -static void -push_samples (GstFFTWSpectrum *conv, GstBuffer *buf) -{ - GstMapInfo map; - gst_buffer_map(buf, &map, GST_MAP_READ); - - gint newsamples = map.size / sizeof (gdouble); - gint oldsamples = conv->numsamples; - - conv->numsamples += newsamples; - conv->samples = g_realloc (conv->samples, conv->numsamples * sizeof (gdouble)); - memcpy (&conv->samples[oldsamples], map.data, - newsamples * sizeof (gdouble)); - - /* GST_LOG ("Added %d samples", newsamples); */ - - gst_buffer_unmap(buf, &map); -} - -/* This basically does the opposite of push_samples, but takes samples - * off the front. - */ -static void -shift_samples (GstFFTWSpectrum *conv, gint toshift) -{ - gdouble *oldsamples = conv->samples; - - conv->numsamples -= toshift; - conv->samples = g_malloc (MAX (conv->numsamples, 1) * sizeof (double)); - memcpy (conv->samples, &oldsamples[toshift], - conv->numsamples * sizeof (gdouble)); - g_free (oldsamples); - - /* Fix the timestamp and offset */ - conv->timestamp - += gst_util_uint64_scale_int (GST_SECOND, toshift, conv->rate); - conv->offset += toshift; - - /* GST_LOG ("Disposed of %d samples (time: %" GST_TIME_FORMAT " offset: %llu)", - toshift, GST_TIME_ARGS(conv->timestamp), conv->offset); */ -} - - -/* This function queues samples until there are at least - * max (conv->size, conv->step) samples to process. We - * then process samples in chunks of conv->size and increment - * by conv->step. - */ -static GstFlowReturn -gst_fftwspectrum_chain(GstPad* pad, GstObject* object, GstBuffer* buf) -{ - GstFFTWSpectrum *conv = GST_FFTWSPECTRUM(object); - GstBuffer *outbuf; - GstFlowReturn res = GST_FLOW_OK; - - push_samples (conv, buf); - gst_buffer_unref (buf); - - GstQuery* query = gst_query_new_allocation(gst_pad_get_current_caps(pad), TRUE); - if (!gst_pad_peer_query(pad, query)) { - // Query failed, not a problem, we use the query defaults. - } - - GstBufferPool* pool = NULL; - guint size = 0; - guint min = 0; - guint max = 0; - if (gst_query_get_n_allocation_pools(query) > 0) { - gst_query_parse_nth_allocation_pool(query, 0, &pool, &size, &min, &max); - } - if (pool == NULL) { - pool = gst_buffer_pool_new(); - } - - GstStructure* config = gst_buffer_pool_get_config(pool); - gst_buffer_pool_config_set_params( - config, gst_pad_get_current_caps(pad), size, min, max); - gst_buffer_pool_set_config(pool, config); - - gst_buffer_pool_set_active(pool, TRUE); - - while (conv->numsamples >= MAX (conv->size, conv->step)) - { - /* - res = gst_pad_alloc_buffer_and_set_caps( - conv->srcpad, - conv->offset, - OUTPUT_SIZE(conv), - GST_PAD_CAPS(conv->srcpad), - &outbuf); - */ - - res = gst_buffer_pool_acquire_buffer(pool, &outbuf, NULL); - - if (res != GST_FLOW_OK) - break; - - gst_buffer_set_size(outbuf, OUTPUT_SIZE(conv)); - GST_BUFFER_OFFSET (outbuf) = conv->offset; - GST_BUFFER_OFFSET_END (outbuf) = conv->offset + conv->step; - GST_BUFFER_TIMESTAMP (outbuf) = conv->timestamp; - GST_BUFFER_DURATION (outbuf) = - gst_util_uint64_scale_int (GST_SECOND, conv->step, conv->rate); - - /* Do the Fourier transform */ - memcpy (conv->fftw_in, conv->samples, conv->size * sizeof (double)); - fftw_execute (conv->fftw_plan); - { /* Normalize */ - gint i; - gfloat root = sqrtf (conv->size); - for (i = 0; i < 2*(conv->size/2+1); ++i) { - conv->fftw_out[i] /= root; - } - } - GstMapInfo map; - gst_buffer_map(outbuf, &map, GST_MAP_WRITE); - memcpy (map.data, conv->fftw_out, OUTPUT_SIZE (conv)); - gst_buffer_unmap(outbuf, &map); - - res = gst_pad_push (conv->srcpad, outbuf); - - shift_samples (conv, conv->step); - - if (res != GST_FLOW_OK) - break; - } - - gst_object_unref (conv); - - return res; -} - diff --git a/gst/moodbar/gstfftwspectrum.h b/gst/moodbar/gstfftwspectrum.h deleted file mode 100644 index f06d2b0e2..000000000 --- a/gst/moodbar/gstfftwspectrum.h +++ /dev/null @@ -1,72 +0,0 @@ -/* GStreamer FFTW-based signal-to-spectrum converter - * Copyright (C) 2006 Joseph Rabinoff - */ - -/*************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ - - -#ifndef __GST_FFTWSPECTRUM_H__ -#define __GST_FFTWSPECTRUM_H__ - -#include -#include - -G_BEGIN_DECLS - -/* #defines don't like whitespacey bits */ -#define GST_TYPE_FFTWSPECTRUM \ - (gst_fftwspectrum_get_type()) -#define GST_FFTWSPECTRUM(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_FFTWSPECTRUM,GstFFTWSpectrum)) -#define GST_FFTWSPECTRUM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_FFTWSPECTRUM,GstFFTWSpectrumClass)) - -typedef struct _GstFFTWSpectrum GstFFTWSpectrum; -typedef struct _GstFFTWSpectrumClass GstFFTWSpectrumClass; - -struct _GstFFTWSpectrum -{ - GstElement element; - - GstPad *sinkpad, *srcpad; - - /* Stream data */ - gint rate, size, step; - - /* Actual queued (incoming) stream */ - gdouble *samples; - gint numsamples; - GstClockTime timestamp; /* Timestamp of the first sample */ - guint64 offset; /* Offset of the first sample */ - - /* State data for fftw */ - double *fftw_in; - double *fftw_out; - fftw_plan fftw_plan; - - /* Properties */ - gint32 def_size, def_step; - gboolean hi_q; - - GMutex* mutex; -}; - -struct _GstFFTWSpectrumClass -{ - GstElementClass parent_class; - - GMutex mutex; -}; - -GType gst_fftwspectrum_get_type (void); - -G_END_DECLS - -#endif /* __GST_FFTWSPECTRUM_H__ */ diff --git a/gst/moodbar/gstmoodbar.c b/gst/moodbar/gstmoodbar.c deleted file mode 100644 index 78fc58c7e..000000000 --- a/gst/moodbar/gstmoodbar.c +++ /dev/null @@ -1,679 +0,0 @@ -/* GStreamer spectrum analysis toy - * Copyright (C) 2006 Joseph Rabinoff - * Some code copyright (C) 2005 Gav Wood - */ - -/*************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ - -/** - * SECTION:element-moodbar - * - * - * Example launch line - * - * - * gst-launch filesrc location=test.mp3 ! mad ! audioconvert ! fftwspectrum ! moodbar height=50 ! pngenc ! filesink location=test.png - * - * - * - */ - -/* This plugin is based on the Moodbar code in Amarok version 1.4.0a, - * written by Gav Wood. The algorithm is basically the same as the - * one applied there, and the normalizing code below is taken directly - * from Gav Wood's Exscalibar package. - */ - -/* This plugin takes a frequency-domain stream, does some simple - * analysis, and returns a string of (unsigned char) rgb triples - * that represent the magnitude of various sections of the stream. - * Since we have to perform some normalization, we queue up all - * of our analysis until we get an EOS event, at which point we - * normalize and do the output. If a max-width is specified, the - * output is scaled down to the desired width if necessary. - */ - -/* More precisely, the analysis performed is as follows: - * (1) the spectrum is broken into 24 parts, called "bark bands" - * (Gav's terminology), as given in bark_bands below - * (2) we compute the size of the first 8 bark bands and store - * that as the "red" component; similarly for blue and green - * (3) after receiving an EOS, we normalize all of the analysis - * done in (1) and (2) and return a stream of rgb triples - * (application/x-raw-rgb) - */ - -#ifdef HAVE_CONFIG_H -# include -#endif - -#include -#include -#include - -#include "gstmoodbar.h" -#include "spectrum.h" - -GST_DEBUG_CATEGORY (gst_moodbar_debug); -#define GST_CAT_DEFAULT gst_moodbar_debug - -/* Filter signals and args */ -enum -{ - /* FILL ME */ - LAST_SIGNAL -}; - -enum -{ - ARG_0, - ARG_HEIGHT, - ARG_MAX_WIDTH -}; - -static GstStaticPadTemplate sink_factory - = GST_STATIC_PAD_TEMPLATE ("sink", - GST_PAD_SINK, - GST_PAD_ALWAYS, - GST_STATIC_CAPS - ( SPECTRUM_FREQ_CAPS ) - ); - -static GstStaticPadTemplate src_factory - = GST_STATIC_PAD_TEMPLATE ("src", - GST_PAD_SRC, - GST_PAD_ALWAYS, - GST_STATIC_CAPS - ( "video/x-raw-rgb, " - "bpp = (int) 24, " - "depth = (int) 24, " - "height = (int) [ 1, MAX ], " - "width = (int) [ 1, MAX ], " - "framerate = (fraction) 0/1" - ) - ); - -G_DEFINE_TYPE(GstMoodbar, gst_moodbar, GST_TYPE_ELEMENT); - -static void gst_moodbar_set_property (GObject *object, guint prop_id, - const GValue *value, GParamSpec *pspec); -static void gst_moodbar_get_property (GObject *object, guint prop_id, - GValue *value, GParamSpec *pspec); - -static gboolean gst_moodbar_set_sink_caps (GstPad *pad, GstCaps *caps); -static gboolean gst_moodbar_sink_event ( - GstPad *pad, GstObject *parent, GstEvent *event); - -static GstFlowReturn gst_moodbar_chain ( - GstPad *pad, GstObject *object, GstBuffer *buf); -static GstStateChangeReturn gst_moodbar_change_state (GstElement *element, - GstStateChange transition); - -static void gst_moodbar_finish (GstMoodbar *mood); - -/* This is a failsafe so we don't eat up all of a computer's memory - * if we hit an endless stream. */ -#define MAX_TRIPLES (1024*1024*4) - -#define NUMFREQS(mood) ((mood)->size/2+1) - -/* Allocate mood->r, mood->g, and mood->b in chunks of this many */ -#define FRAME_CHUNK 1000 - -/* Default height of the output image */ -#define HEIGHT_DEFAULT 1 - -/* Default max-width of the output image, or 0 for no rescaling */ -#define MAX_WIDTH_DEFAULT 0 - -/* We use this table to break up the incoming spectrum into segments */ -static const guint bark_bands[24] - = { 100, 200, 300, 400, 510, 630, 770, 920, - 1080, 1270, 1480, 1720, 2000, 2320, 2700, 3150, - 3700, 4400, 5300, 6400, 7700, 9500, 12000, 15500 }; - - -/***************************************************************/ -/* GObject boilerplate stuff */ -/***************************************************************/ - -/* initialize the plugin's class */ -static void -gst_moodbar_class_init (GstMoodbarClass * klass) -{ - GObjectClass *gobject_class; - GstElementClass *gstelement_class; - - gobject_class = (GObjectClass *) klass; - gstelement_class = (GstElementClass *) klass; - - gobject_class->set_property = gst_moodbar_set_property; - gobject_class->get_property = gst_moodbar_get_property; - - g_object_class_install_property (gobject_class, ARG_HEIGHT, - g_param_spec_int ("height", "Image height", - "The height of the resulting raw image", - 1, G_MAXINT32, HEIGHT_DEFAULT, G_PARAM_READWRITE)); - - g_object_class_install_property (gobject_class, ARG_MAX_WIDTH, - g_param_spec_int ("max-width", "Image maximum width", - "The maximum width of the resulting raw image, or 0 for no rescaling", - 0, G_MAXINT32, MAX_WIDTH_DEFAULT, G_PARAM_READWRITE)); - - gstelement_class->change_state - = GST_DEBUG_FUNCPTR (gst_moodbar_change_state); - - gst_element_class_add_pad_template (gstelement_class, - gst_static_pad_template_get (&src_factory)); - gst_element_class_add_pad_template (gstelement_class, - gst_static_pad_template_get (&sink_factory)); - - gst_element_class_set_static_metadata( - gstelement_class, - "Moodbar analyzer", - "Filter/Converter/Moodbar", - "Convert a spectrum into a stream of (uchar) rgb triples representing its \"mood\"", - "Joe Rabinoff "); -} - -/* initialize the new element - * instantiate pads and add them to element - * set functions - * initialize structure - */ -static void -gst_moodbar_init (GstMoodbar *mood) -{ - GstElementClass *klass = GST_ELEMENT_GET_CLASS (mood); - - mood->sinkpad = - gst_pad_new_from_template - (gst_element_class_get_pad_template (klass, "sink"), "sink"); - gst_pad_set_event_function (mood->sinkpad, - GST_DEBUG_FUNCPTR (gst_moodbar_sink_event)); - gst_pad_set_chain_function (mood->sinkpad, - GST_DEBUG_FUNCPTR (gst_moodbar_chain)); - - mood->srcpad = - gst_pad_new_from_template - (gst_element_class_get_pad_template (klass, "src"), "src"); - - - gst_element_add_pad (GST_ELEMENT (mood), mood->sinkpad); - gst_element_add_pad (GST_ELEMENT (mood), mood->srcpad); - - /* These are set once the (sink) capabilities are determined */ - mood->rate = 0; - mood->size = 0; - mood->barkband_table = NULL; - - /* These are allocated when we change to PAUSED */ - mood->r = NULL; - mood->g = NULL; - mood->b = NULL; - mood->numframes = 0; - - /* Property */ - mood->height = HEIGHT_DEFAULT; - mood->max_width = MAX_WIDTH_DEFAULT; -} - - -static void gst_moodbar_set_property (GObject *object, guint prop_id, - const GValue *value, GParamSpec *pspec) -{ - GstMoodbar *mood = GST_MOODBAR (object); - - switch (prop_id) - { - case ARG_HEIGHT: - mood->height = (guint) g_value_get_int (value); - break; - case ARG_MAX_WIDTH: - mood->max_width = (guint) g_value_get_int (value); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } - -} - - -static void gst_moodbar_get_property (GObject *object, guint prop_id, - GValue *value, GParamSpec *pspec) -{ - GstMoodbar *mood = GST_MOODBAR (object); - - switch (prop_id) - { - case ARG_HEIGHT: - g_value_set_int (value, (int) mood->height); - break; - case ARG_MAX_WIDTH: - g_value_set_int (value, (int) mood->max_width); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - - - - -/***************************************************************/ -/* Pad handling */ -/***************************************************************/ - - -/* This calculates a table that caches which bark band slot each - * incoming band is supposed to go in. */ -static void -calc_barkband_table (GstMoodbar *mood) -{ - guint i; - guint barkband = 0; - - /* Avoid divide-by-zero */ - if (!mood->size || !mood->rate) - return; - - if (mood->barkband_table) - g_free (mood->barkband_table); - - mood->barkband_table = g_malloc (NUMFREQS (mood) * sizeof (guint)); - - for (i = 0; i < NUMFREQS (mood); ++i) - { - if (barkband < 23 && - (guint) GST_SPECTRUM_BAND_FREQ (i, mood->size, mood->rate) - >= bark_bands[barkband]) - barkband++; - - mood->barkband_table[i] = barkband; - - /* - GST_LOG ("Band %d (frequency %f) -> barkband %d (frequency %d)", - i, GST_SPECTRUM_BAND_FREQ (i, mood->size, mood->rate), - barkband, bark_bands[barkband]); - */ - } -} - - -/* Setting the sink caps just gets the rate and size parameters. - * Note that we do not support upstream caps renegotiation, since - * we could only possibly scale the height anyway. - */ - -static gboolean -gst_moodbar_set_sink_caps (GstPad *pad, GstCaps *caps) -{ - GstMoodbar *mood; - GstStructure *newstruct; - gint rate, size; - gboolean res = FALSE; - - mood = GST_MOODBAR (gst_pad_get_parent (pad)); - - newstruct = gst_caps_get_structure (caps, 0); - if (!gst_structure_get_int (newstruct, "rate", &rate) || - !gst_structure_get_int (newstruct, "size", &size)) - goto out; - - res = TRUE; - - mood->rate = rate; - mood->size = (guint) size; - calc_barkband_table (mood); - - out: - gst_object_unref (mood); - - return res; -} - - -static gboolean -gst_moodbar_sink_event (GstPad *pad, GstObject *parent, GstEvent *event) -{ - GstMoodbar *mood = GST_MOODBAR(parent); - gboolean res = TRUE; - - if (GST_EVENT_TYPE (event) == GST_EVENT_EOS) { - gst_moodbar_finish (mood); - } else if (GST_EVENT_TYPE(event) == GST_EVENT_CAPS) { - GstCaps* caps = NULL; - gst_event_parse_caps(event, &caps); - gst_moodbar_set_sink_caps(pad, caps); - } - - res = gst_pad_push_event (mood->srcpad, event); - gst_object_unref (mood); - - return res; -} - - -/***************************************************************/ -/* Actual analysis */ -/***************************************************************/ - - -static GstStateChangeReturn -gst_moodbar_change_state (GstElement *element, GstStateChange transition) -{ - GstMoodbar *mood = GST_MOODBAR (element); - GstStateChangeReturn res; - - switch (transition) - { - case GST_STATE_CHANGE_NULL_TO_READY: - calc_barkband_table (mood); - break; - case GST_STATE_CHANGE_READY_TO_PAUSED: - mood->r = (gdouble *) g_malloc (FRAME_CHUNK * sizeof(gdouble)); - mood->g = (gdouble *) g_malloc (FRAME_CHUNK * sizeof(gdouble)); - mood->b = (gdouble *) g_malloc (FRAME_CHUNK * sizeof(gdouble)); - mood->numframes = 0; - break; - case GST_STATE_CHANGE_PAUSED_TO_PLAYING: - break; - default: - break; - } - - res = GST_ELEMENT_CLASS(gst_moodbar_parent_class) - ->change_state (element, transition); - - switch (transition) - { - case GST_STATE_CHANGE_PLAYING_TO_PAUSED: - break; - case GST_STATE_CHANGE_PAUSED_TO_READY: - g_free (mood->r); - g_free (mood->g); - g_free (mood->b); - mood->r = NULL; - mood->g = NULL; - mood->b = NULL; - mood->numframes = 0; - break; - case GST_STATE_CHANGE_READY_TO_NULL: - g_free (mood->barkband_table); - mood->barkband_table = NULL; - break; - default: - break; - } - - return res; -} - - -/* We allocate r, g, b frames in chunks of FRAME_CHUNK so we don't - * have to realloc every time a buffer comes in. - */ -static gboolean -allocate_another_frame (GstMoodbar *mood) -{ - mood->numframes++; - - /* Failsafe */ - if (mood->numframes == MAX_TRIPLES) - return FALSE; - - if(mood->numframes % FRAME_CHUNK == 0) - { - guint size = (mood->numframes + FRAME_CHUNK) * sizeof (gdouble); - - mood->r = (gdouble *) g_realloc (mood->r, size); - mood->g = (gdouble *) g_realloc (mood->g, size); - mood->b = (gdouble *) g_realloc (mood->b, size); - - if (mood->r == NULL || mood->g == NULL || mood->b == NULL) - return FALSE; - } - - return TRUE; -} - - -/* This function does most of the analysis on the spectra we - * get as input and caches them. We actually push buffers - * once we receive an EOS signal. - */ -static GstFlowReturn -gst_moodbar_chain (GstPad *pad, GstObject *parent, GstBuffer *buf) -{ - GstMoodbar *mood = GST_MOODBAR (parent); - guint i; - gdouble amplitudes[24], rgb[3] = {0.f, 0.f, 0.f}; - gdouble *out, real, imag; - guint numfreqs = NUMFREQS (mood); - - GstMapInfo map; - gst_buffer_map(buf, &map, GST_MAP_READ); - - if (map.size != numfreqs * sizeof (gdouble) * 2) - { - gst_buffer_unmap(buf, &map); - gst_object_unref (mood); - return GST_FLOW_ERROR; - } - - out = (gdouble *) map.data; - - if (!allocate_another_frame (mood)) - return GST_FLOW_ERROR; - - /* Calculate total amplitudes for the different bark bands */ - - for (i = 0; i < 24; ++i) - amplitudes[i] = 0.f; - - for (i = 0; i < numfreqs; ++i) - { - real = out[2*i]; imag = out[2*i + 1]; - amplitudes[mood->barkband_table[i]] += sqrtf (real*real + imag*imag); - } - - /* Now divide the bark bands into thirds and compute their total - * amplitudes */ - - for (i = 0; i < 24; ++i) - rgb[i/8] += amplitudes[i] * amplitudes[i]; - - rgb[0] = sqrtf (rgb[0]); - rgb[1] = sqrtf (rgb[1]); - rgb[2] = sqrtf (rgb[2]); - - mood->r[mood->numframes] = rgb[0]; - mood->g[mood->numframes] = rgb[1]; - mood->b[mood->numframes] = rgb[2]; - - gst_buffer_unmap (buf, &map); - gst_buffer_unref (buf); - gst_object_unref (mood); - - return GST_FLOW_OK; -} - - -/* The normalization code was copied from Gav Wood's Exscalibar - * library, normalise.cpp - */ -static void -normalize (gdouble *vals, guint numvals) -{ - gdouble mini, maxi, tu = 0.f, tb = 0.f; - gdouble avgu = 0.f, avgb = 0.f, delta, avg = 0.f; - gdouble avguu = 0.f, avgbb = 0.f; - guint i; - gint t = 0; - - if (!numvals) - return; - - mini = maxi = vals[0]; - - for (i = 1; i < numvals; i++) - { - if (vals[i] > maxi) - maxi = vals[i]; - else if (vals[i] < mini) - mini = vals[i]; - } - - for (i = 0; i < numvals; i++) - { - if(vals[i] != mini && vals[i] != maxi) - { - avg += vals[i] / ((gdouble) numvals); - t++; - } - } - - for (i = 0; i < numvals; i++) - { - if (vals[i] != mini && vals[i] != maxi) - { - if (vals[i] > avg) - { - avgu += vals[i]; - tu++; - } - else - { - avgb += vals[i]; - tb++; - } - } - } - - avgu /= (gdouble) tu; - avgb /= (gdouble) tb; - - tu = 0.f; - tb = 0.f; - for (i = 0; i < numvals; i++) - { - if (vals[i] != mini && vals[i] != maxi) - { - if (vals[i] > avgu) - { - avguu += vals[i]; - tu++; - } - - else if (vals[i] < avgb) - { - avgbb += vals[i]; - tb++; - } - } - } - - avguu /= (gdouble) tu; - avgbb /= (gdouble) tb; - - mini = MAX (avg + (avgb - avg) * 2.f, avgbb); - maxi = MIN (avg + (avgu - avg) * 2.f, avguu); - delta = maxi - mini; - - if (delta == 0.f) - delta = 1.f; - - for (i = 0; i < numvals; i++) - vals[i] = isfinite (vals[i]) ? MIN(1.f, MAX(0.f, (vals[i] - mini) / delta)) - : 0.f; -} - - - -/* This function normalizes all of the cached r,g,b data and - * finally pushes a monster buffer with all of our output. - */ -static void -gst_moodbar_finish (GstMoodbar *mood) -{ - GstBuffer *buf; - guchar *data; - guint line; - guint output_width; - - if (mood->max_width == 0 - || mood->numframes <= mood->max_width) - output_width = mood->numframes; - else - output_width = mood->max_width; - - normalize (mood->r, mood->numframes); - normalize (mood->g, mood->numframes); - normalize (mood->b, mood->numframes); - - buf = gst_buffer_new_and_alloc - (output_width * mood->height * 3 * sizeof (guchar)); - if (!buf) - return; - - GstMapInfo map; - gst_buffer_map(buf, &map, GST_MAP_READ); - /* Don't set the timestamp, duration, etc. since it's irrelevant */ - map.memory->offset = 0; - data = (guchar *) map.data; - - gdouble r, g, b; - guint i, j, n; - guint start, end; - for (line = 0; line < mood->height; ++line) - { - for (i = 0; i < output_width; ++i) - { - r = 0.f; g = 0.f; b = 0.f; - start = i * mood->numframes / output_width; - end = (i + 1) * mood->numframes / output_width; - if ( start == end ) - end = start + 1; - - for( j = start; j < end; j++ ) - { - r += mood->r[j] * 255.f; - g += mood->g[j] * 255.f; - b += mood->b[j] * 255.f; - } - - n = end - start; - - *(data++) = (guchar) (r / ((gdouble) n)); - *(data++) = (guchar) (g / ((gdouble) n)); - *(data++) = (guchar) (b / ((gdouble) n)); - } - } - - { /* Now we (finally) know the width of the image we're pushing */ - GstCaps *caps = gst_caps_copy (gst_pad_get_current_caps (mood->srcpad)); - gboolean res; - gst_caps_set_simple (caps, "width", G_TYPE_INT, output_width, NULL); - gst_caps_set_simple (caps, "height", G_TYPE_INT, mood->height, NULL); - res = gst_pad_set_caps (mood->srcpad, caps); - /* - if (res) - gst_buffer_set_caps (buf, caps); - */ - gst_caps_unref (caps); - if (!res) - return; - } - - gst_pad_push (mood->srcpad, buf); - gst_buffer_unmap(buf, &map); -} diff --git a/gst/moodbar/gstmoodbar.h b/gst/moodbar/gstmoodbar.h deleted file mode 100644 index 6ce2f799c..000000000 --- a/gst/moodbar/gstmoodbar.h +++ /dev/null @@ -1,63 +0,0 @@ -/* GStreamer spectrum analysis toy - * Copyright (C) 2006 Joseph Rabinoff - * Some code copyright (C) 2005 Gav Wood - */ - -/*************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ - -#ifndef __GST_MOODBAR_H__ -#define __GST_MOODBAR_H__ - -#include - -G_BEGIN_DECLS - -/* #defines don't like whitespacey bits */ -#define GST_TYPE_MOODBAR \ - (gst_moodbar_get_type()) -#define GST_MOODBAR(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_MOODBAR,GstMoodbar)) -#define GST_MOODBAR_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_MOODBAR,GstMoodbarClass)) - -typedef struct _GstMoodbar GstMoodbar; -typedef struct _GstMoodbarClass GstMoodbarClass; - -struct _GstMoodbar -{ - GstElement element; - - GstPad *sinkpad, *srcpad; - - /* Stream data */ - gint rate, size; - - /* Cached band -> bark band table */ - guint *barkband_table; - - /* Queued moodbar data */ - gdouble *r, *g, *b; - guint numframes; - - /* Property */ - guint height; - guint max_width; -}; - -struct _GstMoodbarClass -{ - GstElementClass parent_class; -}; - -GType gst_moodbar_get_type (void); - -G_END_DECLS - -#endif /* __GST_MOODBAR_H__ */ diff --git a/gst/moodbar/plugin.cpp b/gst/moodbar/plugin.cpp new file mode 100644 index 000000000..b70e53b99 --- /dev/null +++ b/gst/moodbar/plugin.cpp @@ -0,0 +1,48 @@ +/* This file is part of Clementine. + Copyright 2014, David Sansome + + Clementine is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Clementine is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Clementine. If not, see . +*/ + +#include + +#include "gstfastspectrum.h" +#include "plugin.h" + +namespace { + +static gboolean plugin_init(GstPlugin* plugin) { + if (!gst_element_register(plugin, "fastspectrum", + GST_RANK_NONE, GST_TYPE_FASTSPECTRUM)) { + return FALSE; + } + + return TRUE; +} + +} // namespace + +int gstfastspectrum_register_static() { + return gst_plugin_register_static( + GST_VERSION_MAJOR, + GST_VERSION_MINOR, + "fastspectrum", + "Fast spectrum analyzer for generating Moodbars", + plugin_init, + "0.1", + "GPL", + "FastSpectrum", + "FastSpectrum", + "https://www.clementine-player.org"); +} diff --git a/gst/moodbar/plugin.h b/gst/moodbar/plugin.h new file mode 100644 index 000000000..456ee2f94 --- /dev/null +++ b/gst/moodbar/plugin.h @@ -0,0 +1,25 @@ +/* This file is part of Clementine. + Copyright 2014, David Sansome + + Clementine is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Clementine is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Clementine. If not, see . +*/ + +#ifndef GST_MOODBAR_PLUGIN_H_ +#define GST_MOODBAR_PLUGIN_H_ + +extern "C" { + int gstfastspectrum_register_static(); +} + +#endif // GST_MOODBAR_PLUGIN_H_ diff --git a/gst/moodbar/spectrum.c b/gst/moodbar/spectrum.c deleted file mode 100644 index f82f3932f..000000000 --- a/gst/moodbar/spectrum.c +++ /dev/null @@ -1,71 +0,0 @@ -/* GStreamer moodbar plugin globals - * Copyright (C) 2006 Joseph Rabinoff - */ - -/*************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ - -#ifdef HAVE_CONFIG_H -# include -#endif - -#include - -#include "gstfftwspectrum.h" -#include "gstmoodbar.h" -#include "spectrum.h" - - -/***************************************************************/ -/* Plugin managing */ -/***************************************************************/ - -GST_DEBUG_CATEGORY_EXTERN (gst_fftwspectrum_debug); -GST_DEBUG_CATEGORY_EXTERN (gst_moodbar_debug); - - -/* entry point to initialize the plug-in - * initialize the plug-in itself - * register the element factories and pad templates - * register the features - * - * exchange the string 'plugin' with your elemnt name - */ -static gboolean -plugin_init (GstPlugin * plugin) -{ - if (!gst_element_register (plugin, "fftwspectrum", - GST_RANK_NONE, GST_TYPE_FFTWSPECTRUM)) - return FALSE; - - if (!gst_element_register (plugin, "moodbar", - GST_RANK_NONE, GST_TYPE_MOODBAR)) - return FALSE; - - GST_DEBUG_CATEGORY_INIT (gst_fftwspectrum_debug, "fftwspectrum", - 0, "FFTW Sample-to-Spectrum Converter Plugin"); - GST_DEBUG_CATEGORY_INIT (gst_moodbar_debug, "moodbar", - 0, "Moodbar analyzer"); - - return TRUE; -} - -void gstmoodbar_register_static() { - gst_plugin_register_static( - GST_VERSION_MAJOR, - GST_VERSION_MINOR, - "moodbar", - "Frequency analyzer and converter plugin", - plugin_init, - "0.1.2", - "GPL", - "Moodbar", - "Moodbar", - "http://amarok.kde.org/wiki/Moodbar"); -} diff --git a/gst/moodbar/spectrum.h b/gst/moodbar/spectrum.h deleted file mode 100644 index 65f271338..000000000 --- a/gst/moodbar/spectrum.h +++ /dev/null @@ -1,67 +0,0 @@ -/* GStreamer moodbar plugin globals - * Copyright (C) 2006 Joseph Rabinoff - */ - -/*************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ - -#ifndef __SPECTRUM_H__ -#define __SPECTRUM_H__ - - -/* Since fftwspectrum and fftwunspectrum are supposed to be - * opposites, they'll be using the same caps: */ - -#define SPECTRUM_SIGNAL_CAPS "audio/x-raw-float, " \ - "rate = (int) [ 1, MAX ], " \ - "channels = (int) 1, " \ - "endianness = (int) BYTE_ORDER, " \ - "width = (int) 64, " \ - "signed = (boolean) true" - -/* audio/x-spectrum-complex-float is an array of complex floats. A - * complex float is just a pair (r, i) of a real float and an - * imaginary float, each with the specified width. The properties - * are as follows: - * rate: the rate of the original signal - * size: the number of signals processed to make the current buffer - * step: the number of signals advanced after the current buffer - * width: the size of the real & imaginary parts of the data - * endianness: ditto - * - * Each audio/x-spectrum-complex-float buffer represents the Fourier - * transform of size samples, and hence _must_ have exactly - * floor(size/2) + 1 complex floats in it; in other words, its - * buffer size must be (floor(size/2) + 1) * 2 * sizeof(gfloat) - */ - -#define SPECTRUM_FREQ_CAPS "audio/x-spectrum-complex-float, " \ - "rate = (int) [ 1, MAX ], " \ - "endianness = (int) BYTE_ORDER, " \ - "width = (int) 64, " \ - "size = (int) [ 1, MAX ], " \ - "step = (int) [ 1, MAX ]" - - -/* Given a band number from a spectrum made from size audio - * samples at the given rate, return the frequency that band - * corresponds to. - */ -#define GST_SPECTRUM_BAND_FREQ(band, size, rate) \ - (((gfloat)(band))*((gfloat)(rate))/((gfloat)(size))) - -#ifdef __cplusplus -extern "C" { -#endif - void gstmoodbar_register_static(); -#ifdef __cplusplus -} -#endif - -#endif /* __SPECTRUM_H__ */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 650fa743a..103c00c05 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1056,6 +1056,7 @@ optional_source(HAVE_LIBMTP # Moodbar support optional_source(HAVE_MOODBAR SOURCES + moodbar/moodbarbuilder.cpp moodbar/moodbarcontroller.cpp moodbar/moodbaritemdelegate.cpp moodbar/moodbarloader.cpp diff --git a/src/engines/gstengine.cpp b/src/engines/gstengine.cpp index 12cde7d47..283b5346f 100755 --- a/src/engines/gstengine.cpp +++ b/src/engines/gstengine.cpp @@ -49,7 +49,7 @@ #include "core/utilities.h" #ifdef HAVE_MOODBAR -#include "gst/moodbar/spectrum.h" +#include "gst/moodbar/plugin.h" #endif #ifdef HAVE_LIBPULSE @@ -128,7 +128,7 @@ void GstEngine::InitialiseGstreamer() { gst_init(nullptr, nullptr); #ifdef HAVE_MOODBAR - gstmoodbar_register_static(); + gstfastspectrum_register_static(); #endif QSet plugin_names; diff --git a/src/moodbar/moodbarbuilder.cpp b/src/moodbar/moodbarbuilder.cpp new file mode 100644 index 000000000..f19386e42 --- /dev/null +++ b/src/moodbar/moodbarbuilder.cpp @@ -0,0 +1,191 @@ +/* This file is part of Clementine. + Copyright 2014, David Sansome + + Clementine is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Clementine is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Clementine. If not, see . +*/ + +#include "moodbarbuilder.h" +#include "core/arraysize.h" + +#include + +namespace { + +static const int sBarkBands[] = { + 100, 200, 300, 400, 510, 630, 770, 920, + 1080, 1270, 1480, 1720, 2000, 2320, 2700, 3150, + 3700, 4400, 5300, 6400, 7700, 9500, 12000, 15500 }; + +static const int sBarkBandCount = arraysize(sBarkBands); + +} // namespace + +MoodbarBuilder::MoodbarBuilder() + : bands_(0), + rate_hz_(0) { +} + +int MoodbarBuilder::BandFrequency(int band) const { + return ((rate_hz_ / 2) * band + rate_hz_ / 4) / bands_; +} + +void MoodbarBuilder::Init(int bands, int rate_hz) { + bands_ = bands; + rate_hz_ = rate_hz; + + barkband_table_.clear(); + barkband_table_.reserve(bands + 1); + + int barkband = 0; + for (int i = 0; i < bands + 1; ++i) { + if (barkband < sBarkBandCount - 1 && + BandFrequency(i) >= sBarkBands[barkband]) { + barkband++; + } + + barkband_table_.append(barkband); + } +} + +void MoodbarBuilder::AddFrame(const double* magnitudes, int size) { + if (size > barkband_table_.length()) { + return; + } + + // Calculate total magnitudes for different bark bands. + double bands[sBarkBandCount]; + for (int i = 0; i < sBarkBandCount; ++i) { + bands[i] = 0.0; + } + + for (int i = 0; i < size; ++i) { + bands[barkband_table_[i]] += magnitudes[i]; + } + + // Now divide the bark bands into thirds and compute their total amplitudes. + double rgb[] = {0, 0, 0}; + for (int i = 0; i < sBarkBandCount; ++i) { + rgb[(i * 3) / sBarkBandCount] += bands[i] * bands[i]; + } + + frames_.append(Rgb(sqrt(rgb[0]), sqrt(rgb[1]), sqrt(rgb[2]))); +} + +void MoodbarBuilder::Normalize(QList* vals, double Rgb::*member) { + double mini = vals->at(0).*member; + double maxi = vals->at(0).*member; + for (int i = 1; i < vals->count(); i++) { + const double value = vals->at(i).*member; + if (value > maxi) { + maxi = value; + } else if (value < mini) { + mini = value; + } + } + + double avg = 0; + int t = 0; + for (const Rgb& rgb : *vals) { + const double value = rgb.*member; + if (value != mini && value != maxi) { + avg += value / vals->count(); + t++; + } + } + + double tu = 0; + double tb = 0; + double avgu = 0; + double avgb = 0; + for (const Rgb& rgb : *vals) { + const double value = rgb.*member; + if (value != mini && value != maxi) { + if (value > avg) { + avgu += value; + tu++; + } else { + avgb += value; + tb++; + } + } + } + avgu /= tu; + avgb /= tb; + + tu = 0; + tb = 0; + double avguu = 0; + double avgbb = 0; + for (const Rgb& rgb : *vals) { + const double value = rgb.*member; + if (value != mini && value != maxi) { + if (value > avgu) { + avguu += value; + tu++; + } else if (value < avgb) { + avgbb += value; + tb++; + } + } + } + avguu /= tu; + avgbb /= tb; + + mini = std::max(avg + (avgb - avg) * 2, avgbb); + maxi = std::min(avg + (avgu - avg) * 2, avguu); + double delta = maxi - mini; + if (delta == 0) { + delta = 1; + } + + for (auto it = vals->begin(); it != vals->end(); ++it) { + double* value = &((*it).*member); + *value = std::isfinite(*value) + ? qBound(0.0, (*value - mini) / delta, 1.0) + : 0; + } +} + +QByteArray MoodbarBuilder::Finish(int width) { + Normalize(&frames_, &Rgb::r); + Normalize(&frames_, &Rgb::g); + Normalize(&frames_, &Rgb::b); + + QByteArray ret; + ret.resize(width * 3); + char* data = ret.data(); + + for (int i = 0; i < width; ++i) { + Rgb rgb; + int start = i * frames_.count() / width; + int end = (i + 1) * frames_.count() / width; + if (start == end) { + end = start + 1; + } + + for (int j = start; j < end; j++) { + const Rgb& frame = frames_[j]; + rgb.r += frame.r * 255; + rgb.g += frame.g * 255; + rgb.b += frame.b * 255; + } + + const int n = end - start; + + *(data++) = rgb.r / n; + *(data++) = rgb.g / n; + *(data++) = rgb.b / n; + } + return ret; +} diff --git a/src/moodbar/moodbarbuilder.h b/src/moodbar/moodbarbuilder.h new file mode 100644 index 000000000..c718ff6cc --- /dev/null +++ b/src/moodbar/moodbarbuilder.h @@ -0,0 +1,50 @@ +/* This file is part of Clementine. + Copyright 2014, David Sansome + + Clementine is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Clementine is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Clementine. If not, see . +*/ + +#ifndef MOODBARBUILDER_H +#define MOODBARBUILDER_H + +#include +#include + +class MoodbarBuilder { + public: + MoodbarBuilder(); + + void Init(int bands, int rate_hz); + void AddFrame(const double* magnitudes, int size); + QByteArray Finish(int width); + + private: + struct Rgb { + Rgb() : r(0), g(0), b(0) {} + Rgb(double r_, double g_, double b_) : r(r_), g(g_), b(b_) {} + + double r, g, b; + }; + + int BandFrequency(int band) const; + static void Normalize(QList* vals, double Rgb::*member); + + QList barkband_table_; + int bands_; + int rate_hz_; + + QList frames_; +}; + +#endif // MOODBARBUILDER_H diff --git a/src/moodbar/moodbarpipeline.cpp b/src/moodbar/moodbarpipeline.cpp index 43905aa2a..9f3bee927 100644 --- a/src/moodbar/moodbarpipeline.cpp +++ b/src/moodbar/moodbarpipeline.cpp @@ -23,9 +23,14 @@ #include "core/logging.h" #include "core/signalchecker.h" +#include "core/timeconstants.h" #include "core/utilities.h" +#include "moodbar/moodbarbuilder.h" + +#include "gst/moodbar/gstfastspectrum.h" bool MoodbarPipeline::sIsAvailable = false; +const int MoodbarPipeline::kBands = 128; MoodbarPipeline::MoodbarPipeline(const QUrl& local_filename) : QObject(nullptr), @@ -44,12 +49,6 @@ bool MoodbarPipeline::IsAvailable() { } gst_object_unref(factory); - factory = gst_element_factory_find("moodbar"); - if (!factory) { - return false; - } - gst_object_unref(factory); - sIsAvailable = true; } @@ -82,40 +81,44 @@ void MoodbarPipeline::Start() { GstElement* decodebin = CreateElement("uridecodebin"); convert_element_ = CreateElement("audioconvert"); - GstElement* fftwspectrum = CreateElement("fftwspectrum"); - GstElement* moodbar = CreateElement("moodbar"); - GstElement* appsink = CreateElement("appsink"); + GstElement* spectrum = CreateElement("fastspectrum"); + GstElement* fakesink = CreateElement("fakesink"); - if (!decodebin || !convert_element_ || !fftwspectrum || !moodbar || - !appsink) { + if (!decodebin || !convert_element_ || !spectrum || !fakesink) { pipeline_ = nullptr; emit Finished(false); return; } // Join them together - gst_element_link_many(convert_element_, fftwspectrum, moodbar, appsink, - nullptr); + if (!gst_element_link(convert_element_, spectrum) || + !gst_element_link(spectrum, fakesink)) { + qLog(Error) << "Failed to link elements"; + pipeline_ = nullptr; + emit Finished(false); + return; + } + + builder_.reset(new MoodbarBuilder); // Set properties - g_object_set(decodebin, "uri", local_filename_.toEncoded().constData(), + g_object_set(decodebin, + "uri", local_filename_.toEncoded().constData(), nullptr); - g_object_set(fftwspectrum, "def-size", 2048, "def-step", 1024, "hiquality", - true, nullptr); - g_object_set(moodbar, "height", 1, "max-width", 1000, nullptr); + g_object_set(spectrum, + "bands", kBands, + nullptr); + + GstFastSpectrum* fast_spectrum = GST_FASTSPECTRUM(spectrum); + fast_spectrum->output_callback = [this](double* magnitudes, int size) { + builder_->AddFrame(magnitudes, size); + }; // Connect signals CHECKED_GCONNECT(decodebin, "pad-added", &NewPadCallback, this); - gst_bus_set_sync_handler(gst_pipeline_get_bus(GST_PIPELINE(pipeline_)), - BusCallbackSync, this, nullptr); - - // Set appsink callbacks - GstAppSinkCallbacks callbacks; - memset(&callbacks, 0, sizeof(callbacks)); - callbacks.new_sample = NewBufferCallback; - - gst_app_sink_set_callbacks(reinterpret_cast(appsink), &callbacks, - this, nullptr); + GstBus* bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline_)); + gst_bus_set_sync_handler(bus, BusCallbackSync, this, nullptr); + gst_object_unref(bus); // Start playing gst_element_set_state(pipeline_, GST_STATE_PLAYING); @@ -146,21 +149,14 @@ void MoodbarPipeline::NewPadCallback(GstElement*, GstPad* pad, gpointer data) { gst_pad_link(pad, audiopad); gst_object_unref(audiopad); -} -GstFlowReturn MoodbarPipeline::NewBufferCallback(GstAppSink* app_sink, - gpointer data) { - MoodbarPipeline* self = reinterpret_cast(data); + int rate = 0; + GstCaps* caps = gst_pad_get_current_caps(pad); + GstStructure* structure = gst_caps_get_structure(caps, 0); + gst_structure_get_int(structure, "rate", &rate); + gst_caps_unref(caps); - GstSample* sample = gst_app_sink_pull_sample(app_sink); - GstBuffer* buffer = gst_sample_get_buffer(sample); - GstMapInfo map; - gst_buffer_map(buffer, &map, GST_MAP_READ); - self->data_.append(reinterpret_cast(map.data), map.size); - gst_buffer_unmap(buffer, &map); - gst_buffer_unref(buffer); - - return GST_FLOW_OK; + self->builder_->Init(kBands, rate); } GstBusSyncReply MoodbarPipeline::BusCallbackSync(GstBus*, GstMessage* msg, @@ -185,6 +181,9 @@ GstBusSyncReply MoodbarPipeline::BusCallbackSync(GstBus*, GstMessage* msg, void MoodbarPipeline::Stop(bool success) { success_ = success; + data_ = builder_->Finish(1000); + builder_.reset(); + emit Finished(success); } diff --git a/src/moodbar/moodbarpipeline.h b/src/moodbar/moodbarpipeline.h index 63374176e..c7acad8e5 100644 --- a/src/moodbar/moodbarpipeline.h +++ b/src/moodbar/moodbarpipeline.h @@ -24,6 +24,10 @@ #include #include +#include + +class MoodbarBuilder; + // Creates moodbar data for a single local music file. class MoodbarPipeline : public QObject { Q_OBJECT @@ -40,7 +44,7 @@ class MoodbarPipeline : public QObject { public slots: void Start(); -signals: + signals: void Finished(bool success); private: @@ -58,11 +62,14 @@ signals: private: static bool sIsAvailable; + static const int kBands; QUrl local_filename_; GstElement* pipeline_; GstElement* convert_element_; + std::unique_ptr builder_; + bool success_; QByteArray data_; };