Funkwhale
/
Clementine-audio-player-Mac-Linux-Windows
mirror of https://github.com/clementine-player/Clementine
1
0
Fork 0
Code Issues Releases Wiki Activity

Rewrite moodbar stuff for gstreamer-1.0: 

- Rewrite gstspectrum (1.0) to use FFTW (2x faster) and emit raw magnitude
  values (not log scaled).
- Rewrite the moodbar generation code to be somewhat understandable, and
  do it in Clementine instead of gstreamer.
This commit is contained in:
David Sansome 2014-09-21 19:35:21 +10:00
parent 6bb81328e8
commit 50551d987a
19 changed files with 1002 additions and 1772 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CMakeLists.txt
View File

@ -53,11 +53,12 @@ pkg_check_modules(CHROMAPRINT libchromaprint)
pkg_check_modules(GIO gio-2.0)
pkg_check_modules(GLIB glib-2.0)
pkg_check_modules(GOBJECT gobject-2.0)
pkg_check_modules(GSTREAMER gstreamer-1.0)
pkg_check_modules(GSTREAMER_APP gstreamer-app-1.0)
pkg_check_modules(GSTREAMER_BASE gstreamer-base-1.0)
pkg_check_modules(GSTREAMER REQUIRED gstreamer-1.0)
pkg_check_modules(GSTREAMER_APP REQUIRED gstreamer-app-1.0)
pkg_check_modules(GSTREAMER_AUDIO REQUIRED gstreamer-audio-1.0)
pkg_check_modules(GSTREAMER_BASE REQUIRED gstreamer-base-1.0)
#pkg_check_modules(GSTREAMER_CDDA gstreamer-cdda-0.10)
pkg_check_modules(GSTREAMER_TAG gstreamer-tag-1.0)
pkg_check_modules(GSTREAMER_TAG REQUIRED gstreamer-tag-1.0)
pkg_check_modules(INDICATEQT indicate-qt)
pkg_check_modules(LIBGPOD libgpod-1.0>=0.7.92)
pkg_check_modules(LIBMTP libmtp>=1.0)
@ -144,6 +145,7 @@ include_directories(${TAGLIB_INCLUDE_DIRS})
include_directories(${QJSON_INCLUDE_DIRS})
include_directories(${GSTREAMER_INCLUDE_DIRS})
include_directories(${GSTREAMER_APP_INCLUDE_DIRS})
include_directories(${GSTREAMER_AUDIO_INCLUDE_DIRS})
include_directories(${GSTREAMER_BASE_INCLUDE_DIRS})
include_directories(${GSTREAMER_CDDA_INCLUDE_DIRS})
include_directories(${GSTREAMER_TAG_INCLUDE_DIRS})

31
gst/moodbar/01-fftw-plan-mutex.patch
View File

@ -1,31 +0,0 @@
From 0b62ebc38d1cc0202c6f57c4e096fa0b68a41baf Mon Sep 17 00:00:00 2001
From: David Sansome <me@davidsansome.com>
Date: Sun, 27 May 2012 17:00:32 +0100
Subject: [PATCH] Protect calls to fftwf_plan_dft_r2c_1d with a mutex
---
plugin/gstfftwspectrum.c | 4 ++++
1 files changed, 4 insertions(+), 0 deletions(-)
diff --git a/plugin/gstfftwspectrum.c b/plugin/gstfftwspectrum.c
index 147e606..f6e2427 100644
--- a/plugin/gstfftwspectrum.c
+++ b/plugin/gstfftwspectrum.c
@@ -302,10 +302,14 @@ alloc_fftw_data (GstFFTWSpectrum *conv)
* outputs are the hermetian conjugates). This should be optimal for
* implementing filters.
*/
+
+ static GStaticMutex mutex = G_STATIC_MUTEX_INIT;
+ g_static_mutex_lock(&mutex);
conv->fftw_plan
= fftwf_plan_dft_r2c_1d(conv->size, conv->fftw_in,
(fftwf_complex *) conv->fftw_out,
conv->hi_q ? FFTW_MEASURE : FFTW_ESTIMATE);
+ g_static_mutex_unlock(&mutex);
}
--
1.7.5.4

8
gst/moodbar/CMakeLists.txt
View File

@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 2.6)
set(CMAKE_C_FLAGS "-Wall")
set(CMAKE_CXX_FLAGS "-Woverloaded-virtual -Wall")
set(CMAKE_CXX_FLAGS "-Woverloaded-virtual -Wall --std=c++0x")
include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
@ -11,9 +11,8 @@ include_directories(${GSTREAMER_INCLUDE_DIRS})
include_directories(${FFTW3_INCLUDE_DIR})
set(SOURCES
gstfftwspectrum.c
gstmoodbar.c
spectrum.c
gstfastspectrum.cpp
plugin.cpp
)
add_library(gstmoodbar STATIC
@ -24,6 +23,7 @@ target_link_libraries(gstmoodbar
${GOBJECT_LIBRARIES}
${GLIB_LIBRARIES}
${GSTREAMER_LIBRARIES}
${GSTREAMER_AUDIO_LIBRARIES}
${GSTREAMER_BASE_LIBRARIES}
${FFTW3_FFTW_LIBRARY}
)

535
gst/moodbar/gstfastspectrum.cpp Normal file
View File

@ -0,0 +1,535 @@
/* GStreamer
* Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
* <2006,2011> Stefan Kost <ensonic@users.sf.net>
* <2007-2009> Sebastian Dröge <sebastian.droege@collabora.co.uk>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#include <cstring>
#include <cmath>
#include "gstfastspectrum.h"
GST_DEBUG_CATEGORY_STATIC (gst_fastspectrum_debug);
#define GST_CAT_DEFAULT gst_fastspectrum_debug
/* elementfactory information */
#if G_BYTE_ORDER == G_LITTLE_ENDIAN
# define FORMATS "{ S16LE, S24LE, S32LE, F32LE, F64LE }"
#else
# define FORMATS "{ S16BE, S24BE, S32BE, F32BE, F64BE }"
#endif
#define ALLOWED_CAPS \
GST_AUDIO_CAPS_MAKE (FORMATS) ", " \
"layout = (string) interleaved, " \
"channels = 1"
/* Spectrum properties */
#define DEFAULT_INTERVAL (GST_SECOND / 10)
#define DEFAULT_BANDS 128
enum {
PROP_0,
PROP_INTERVAL,
PROP_BANDS
};
#define gst_fastspectrum_parent_class parent_class
G_DEFINE_TYPE (GstFastSpectrum, gst_fastspectrum, GST_TYPE_AUDIO_FILTER);
static void gst_fastspectrum_finalize (GObject * object);
static void gst_fastspectrum_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec);
static void gst_fastspectrum_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec);
static gboolean gst_fastspectrum_start (GstBaseTransform * trans);
static gboolean gst_fastspectrum_stop (GstBaseTransform * trans);
static GstFlowReturn gst_fastspectrum_transform_ip (GstBaseTransform * trans,
GstBuffer * in);
static gboolean gst_fastspectrum_setup (GstAudioFilter * base,
const GstAudioInfo * info);
static void
gst_fastspectrum_class_init (GstFastSpectrumClass * klass)
{
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
GstBaseTransformClass *trans_class = GST_BASE_TRANSFORM_CLASS (klass);
GstAudioFilterClass *filter_class = GST_AUDIO_FILTER_CLASS (klass);
GstCaps *caps;
gobject_class->set_property = gst_fastspectrum_set_property;
gobject_class->get_property = gst_fastspectrum_get_property;
gobject_class->finalize = gst_fastspectrum_finalize;
trans_class->start = GST_DEBUG_FUNCPTR (gst_fastspectrum_start);
trans_class->stop = GST_DEBUG_FUNCPTR (gst_fastspectrum_stop);
trans_class->transform_ip = GST_DEBUG_FUNCPTR (gst_fastspectrum_transform_ip);
trans_class->passthrough_on_same_caps = TRUE;
filter_class->setup = GST_DEBUG_FUNCPTR (gst_fastspectrum_setup);
g_object_class_install_property (gobject_class, PROP_INTERVAL,
g_param_spec_uint64 ("interval", "Interval",
"Interval of time between message posts (in nanoseconds)",
1, G_MAXUINT64, DEFAULT_INTERVAL,
GParamFlags(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_BANDS,
g_param_spec_uint ("bands", "Bands", "Number of frequency bands",
0, G_MAXUINT, DEFAULT_BANDS,
GParamFlags(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
GST_DEBUG_CATEGORY_INIT (gst_fastspectrum_debug, "spectrum", 0,
"audio spectrum analyser element");
gst_element_class_set_static_metadata (element_class, "Spectrum analyzer",
"Filter/Analyzer/Audio",
"Run an FFT on the audio signal, output spectrum data",
"Erik Walthinsen <omega@cse.ogi.edu>, "
"Stefan Kost <ensonic@users.sf.net>, "
"Sebastian Dröge <sebastian.droege@collabora.co.uk>");
caps = gst_caps_from_string (ALLOWED_CAPS);
gst_audio_filter_class_add_pad_templates (filter_class, caps);
gst_caps_unref (caps);
}
static void
gst_fastspectrum_init (GstFastSpectrum * spectrum)
{
spectrum->interval = DEFAULT_INTERVAL;
spectrum->bands = DEFAULT_BANDS;
spectrum->channel_data_initialised = false;
g_mutex_init (&spectrum->lock);
}
static void
gst_fastspectrum_alloc_channel_data (GstFastSpectrum * spectrum)
{
guint bands = spectrum->bands;
guint nfft = 2 * bands - 2;
spectrum->input_ring_buffer = new double[nfft];
spectrum->fft_input = reinterpret_cast<double*>(
fftw_malloc(sizeof(double) * nfft));
spectrum->fft_output =reinterpret_cast<fftw_complex*>(
fftw_malloc(sizeof(fftw_complex) * (nfft/2+1)));
spectrum->spect_magnitude = new double[bands];
spectrum->plan = fftw_plan_dft_r2c_1d(
nfft,
spectrum->fft_input,
spectrum->fft_output,
FFTW_ESTIMATE);
spectrum->channel_data_initialised = true;
}
static void
gst_fastspectrum_free_channel_data (GstFastSpectrum * spectrum)
{
if (spectrum->channel_data_initialised) {
fftw_destroy_plan(spectrum->plan);
fftw_free(spectrum->fft_input);
fftw_free(spectrum->fft_output);
delete[] spectrum->input_ring_buffer;
delete[] spectrum->spect_magnitude;
spectrum->channel_data_initialised = false;
}
}
static void
gst_fastspectrum_flush (GstFastSpectrum * spectrum)
{
spectrum->num_frames = 0;
spectrum->num_fft = 0;
spectrum->accumulated_error = 0;
}
static void
gst_fastspectrum_reset_state (GstFastSpectrum * spectrum)
{
GST_DEBUG_OBJECT (spectrum, "resetting state");
gst_fastspectrum_free_channel_data (spectrum);
gst_fastspectrum_flush (spectrum);
}
static void
gst_fastspectrum_finalize (GObject * object)
{
GstFastSpectrum *spectrum = GST_FASTSPECTRUM (object);
gst_fastspectrum_reset_state (spectrum);
g_mutex_clear (&spectrum->lock);
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static void
gst_fastspectrum_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstFastSpectrum *filter = GST_FASTSPECTRUM (object);
switch (prop_id) {
case PROP_INTERVAL:{
guint64 interval = g_value_get_uint64 (value);
g_mutex_lock (&filter->lock);
if (filter->interval != interval) {
filter->interval = interval;
gst_fastspectrum_reset_state (filter);
}
g_mutex_unlock (&filter->lock);
break;
}
case PROP_BANDS:{
guint bands = g_value_get_uint (value);
g_mutex_lock (&filter->lock);
if (filter->bands != bands) {
filter->bands = bands;
gst_fastspectrum_reset_state (filter);
}
g_mutex_unlock (&filter->lock);
break;
}
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_fastspectrum_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstFastSpectrum *filter = GST_FASTSPECTRUM (object);
switch (prop_id) {
case PROP_INTERVAL:
g_value_set_uint64 (value, filter->interval);
break;
case PROP_BANDS:
g_value_set_uint (value, filter->bands);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static gboolean
gst_fastspectrum_start (GstBaseTransform * trans)
{
GstFastSpectrum *spectrum = GST_FASTSPECTRUM (trans);
gst_fastspectrum_reset_state (spectrum);
return TRUE;
}
static gboolean
gst_fastspectrum_stop (GstBaseTransform * trans)
{
GstFastSpectrum *spectrum = GST_FASTSPECTRUM (trans);
gst_fastspectrum_reset_state (spectrum);
return TRUE;
}
/* mixing data readers */
static void
input_data_mixed_float(const guint8* _in, double* out, guint len,
double max_value, guint op, guint nfft)
{
guint j, ip = 0;
gfloat *in = (gfloat *) _in;
for (j = 0; j < len; j++) {
out[op] = in[ip++];
op = (op + 1) % nfft;
}
}
static void
input_data_mixed_double (const guint8 * _in, double* out, guint len,
double max_value, guint op, guint nfft)
{
guint j, ip = 0;
gdouble *in = (gdouble *) _in;
for (j = 0; j < len; j++) {
out[op] = in[ip++];
op = (op + 1) % nfft;
}
}
static void
input_data_mixed_int32_max (const guint8 * _in, double* out, guint len,
double max_value, guint op, guint nfft)
{
guint j, ip = 0;
gint32 *in = (gint32 *) _in;
for (j = 0; j < len; j++) {
out[op] = in[ip++] / max_value;
op = (op + 1) % nfft;
}
}
static void
input_data_mixed_int24_max (const guint8 * _in, double* out, guint len,
double max_value, guint op, guint nfft)
{
guint j;
for (j = 0; j < len; j++) {
#if G_BYTE_ORDER == G_BIG_ENDIAN
gint32 value = GST_READ_UINT24_BE (_in);
#else
gint32 value = GST_READ_UINT24_LE (_in);
#endif
if (value & 0x00800000)
value |= 0xff000000;
out[op] = value / max_value;
op = (op + 1) % nfft;
_in += 3;
}
}
static void
input_data_mixed_int16_max (const guint8 * _in, double * out, guint len,
double max_value, guint op, guint nfft)
{
guint j, ip = 0;
gint16 *in = (gint16 *) _in;
for (j = 0; j < len; j++) {
out[op] = in[ip++] / max_value;
op = (op + 1) % nfft;
}
}
static gboolean
gst_fastspectrum_setup (GstAudioFilter * base, const GstAudioInfo * info)
{
GstFastSpectrum *spectrum = GST_FASTSPECTRUM (base);
GstFastSpectrumInputData input_data = NULL;
g_mutex_lock (&spectrum->lock);
switch (GST_AUDIO_INFO_FORMAT (info)) {
case GST_AUDIO_FORMAT_S16:
input_data = input_data_mixed_int16_max;
break;
case GST_AUDIO_FORMAT_S24:
input_data = input_data_mixed_int24_max;
break;
case GST_AUDIO_FORMAT_S32:
input_data = input_data_mixed_int32_max;
break;
case GST_AUDIO_FORMAT_F32:
input_data = input_data_mixed_float;
break;
case GST_AUDIO_FORMAT_F64:
input_data = input_data_mixed_double;
break;
default:
g_assert_not_reached ();
break;
}
spectrum->input_data = input_data;
gst_fastspectrum_reset_state (spectrum);
g_mutex_unlock (&spectrum->lock);
return TRUE;
}
static void
gst_fastspectrum_run_fft (GstFastSpectrum * spectrum, guint input_pos)
{
guint i;
guint bands = spectrum->bands;
guint nfft = 2 * bands - 2;
for (i = 0; i < nfft; i++)
spectrum->fft_input[i] =
spectrum->input_ring_buffer[(input_pos + i) % nfft];
fftw_execute(spectrum->plan);
gdouble val;
/* Calculate magnitude in db */
for (i = 0; i < bands; i++) {
val = spectrum->fft_output[i][0] * spectrum->fft_output[i][0];
val += spectrum->fft_output[i][1] * spectrum->fft_output[i][1];
val /= nfft * nfft;
spectrum->spect_magnitude[i] += val;
}
}
static GstFlowReturn
gst_fastspectrum_transform_ip (GstBaseTransform * trans, GstBuffer * buffer)
{
GstFastSpectrum *spectrum = GST_FASTSPECTRUM (trans);
guint rate = GST_AUDIO_FILTER_RATE (spectrum);
guint bps = GST_AUDIO_FILTER_BPS (spectrum);
guint bpf = GST_AUDIO_FILTER_BPF (spectrum);
double max_value = (1UL << ((bps << 3) - 1)) - 1;
guint bands = spectrum->bands;
guint nfft = 2 * bands - 2;
guint input_pos;
GstMapInfo map;
const guint8 *data;
gsize size;
guint fft_todo, msg_todo, block_size;
gboolean have_full_interval;
GstFastSpectrumInputData input_data;
g_mutex_lock (&spectrum->lock);
gst_buffer_map (buffer, &map, GST_MAP_READ);
data = map.data;
size = map.size;
GST_LOG_OBJECT (spectrum, "input size: %" G_GSIZE_FORMAT " bytes", size);
if (GST_BUFFER_IS_DISCONT (buffer)) {
GST_DEBUG_OBJECT (spectrum, "Discontinuity detected -- flushing");
gst_fastspectrum_flush (spectrum);
}
/* If we don't have a FFT context yet (or it was reset due to parameter
* changes) get one and allocate memory for everything
*/
if (!spectrum->channel_data_initialised) {
GST_DEBUG_OBJECT (spectrum, "allocating for bands %u", bands);
gst_fastspectrum_alloc_channel_data (spectrum);
/* number of sample frames we process before posting a message
* interval is in ns */
spectrum->frames_per_interval =
gst_util_uint64_scale (spectrum->interval, rate, GST_SECOND);
spectrum->frames_todo = spectrum->frames_per_interval;
/* rounding error for frames_per_interval in ns,
* aggregated it in accumulated_error */
spectrum->error_per_interval = (spectrum->interval * rate) % GST_SECOND;
if (spectrum->frames_per_interval == 0)
spectrum->frames_per_interval = 1;
GST_INFO_OBJECT (spectrum, "interval %" GST_TIME_FORMAT ", fpi %"
G_GUINT64_FORMAT ", error %" GST_TIME_FORMAT,
GST_TIME_ARGS (spectrum->interval), spectrum->frames_per_interval,
GST_TIME_ARGS (spectrum->error_per_interval));
spectrum->input_pos = 0;
gst_fastspectrum_flush (spectrum);
}
if (spectrum->num_frames == 0)
spectrum->message_ts = GST_BUFFER_TIMESTAMP (buffer);
input_pos = spectrum->input_pos;
input_data = spectrum->input_data;
while (size >= bpf) {
/* run input_data for a chunk of data */
fft_todo = nfft - (spectrum->num_frames % nfft);
msg_todo = spectrum->frames_todo - spectrum->num_frames;
GST_LOG_OBJECT (spectrum,
"message frames todo: %u, fft frames todo: %u, input frames %"
G_GSIZE_FORMAT, msg_todo, fft_todo, (size / bpf));
block_size = msg_todo;
if (block_size > (size / bpf))
block_size = (size / bpf);
if (block_size > fft_todo)
block_size = fft_todo;
/* Move the current frames into our ringbuffers */
input_data(data, spectrum->input_ring_buffer, block_size, max_value, input_pos, nfft);
data += block_size * bpf;
size -= block_size * bpf;
input_pos = (input_pos + block_size) % nfft;
spectrum->num_frames += block_size;
have_full_interval = (spectrum->num_frames == spectrum->frames_todo);
GST_LOG_OBJECT (spectrum,
"size: %" G_GSIZE_FORMAT ", do-fft = %d, do-message = %d", size,
(spectrum->num_frames % nfft == 0), have_full_interval);
/* If we have enough frames for an FFT or we have all frames required for
* the interval and we haven't run a FFT, then run an FFT */
if ((spectrum->num_frames % nfft == 0) ||
(have_full_interval && !spectrum->num_fft)) {
gst_fastspectrum_run_fft (spectrum, input_pos);
spectrum->num_fft++;
}
/* Do we have the FFTs for one interval? */
if (have_full_interval) {
GST_DEBUG_OBJECT (spectrum, "nfft: %u frames: %" G_GUINT64_FORMAT
" fpi: %" G_GUINT64_FORMAT " error: %" GST_TIME_FORMAT, nfft,
spectrum->num_frames, spectrum->frames_per_interval,
GST_TIME_ARGS (spectrum->accumulated_error));
spectrum->frames_todo = spectrum->frames_per_interval;
if (spectrum->accumulated_error >= GST_SECOND) {
spectrum->accumulated_error -= GST_SECOND;
spectrum->frames_todo++;
}
spectrum->accumulated_error += spectrum->error_per_interval;
if (spectrum->output_callback) {
// Calculate average
for (uint i = 0; i < spectrum->bands; i++) {
spectrum->spect_magnitude[i] /= spectrum->num_fft;
}
spectrum->output_callback(spectrum->spect_magnitude, spectrum->bands);
// Reset spectrum accumulators
memset(spectrum->spect_magnitude, 0, spectrum->bands * sizeof(double));
}
if (GST_CLOCK_TIME_IS_VALID (spectrum->message_ts))
spectrum->message_ts +=
gst_util_uint64_scale (spectrum->num_frames, GST_SECOND, rate);
spectrum->num_frames = 0;
spectrum->num_fft = 0;
}
}
spectrum->input_pos = input_pos;
gst_buffer_unmap (buffer, &map);
g_mutex_unlock (&spectrum->lock);
g_assert (size == 0);
return GST_FLOW_OK;
}

93
gst/moodbar/gstfastspectrum.h Normal file
View File

@ -0,0 +1,93 @@
/* GStreamer
* Copyright (C) <1999> Erik Walthinsen <omega@cse.ogi.edu>
* Copyright (C) <2009> Sebastian Dröge <sebastian.droege@collabora.co.uk>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
// Adapted from gstspectrum for Clementine with the following changes:
// - Uses fftw instead of kiss fft (2x faster).
// - Hardcoded to 1 channel (use an audioconvert element to do the work
// instead, simplifies this code a lot).
// - Send output via a callback instead of GST messages (less overhead).
// - Removed all properties except interval and band.
#ifndef GST_MOODBAR_FASTSPECTRUM_H_
#define GST_MOODBAR_FASTSPECTRUM_H_
#include <functional>
#include <gst/gst.h>
#include <gst/audio/gstaudiofilter.h>
#include <fftw3.h>
G_BEGIN_DECLS
#define GST_TYPE_FASTSPECTRUM (gst_fastspectrum_get_type())
#define GST_FASTSPECTRUM(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_FASTSPECTRUM,GstFastSpectrum))
#define GST_IS_FASTSPECTRUM(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_FASTSPECTRUM))
#define GST_FASTSPECTRUM_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_FASTSPECTRUM,GstFastSpectrumClass))
#define GST_IS_FASTSPECTRUM_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_FASTSPECTRUM))
using GstFastSpectrumInputData = void(*)(const guint8* in, double* out,
guint len, double max_value, guint op, guint nfft);
using OutputCallback = std::function<void(double* magnitudes, int size)>;
struct GstFastSpectrum {
GstAudioFilter parent;
/* properties */
guint64 interval; /* how many nanoseconds between emits */
guint64 frames_per_interval; /* how many frames per interval */
guint64 frames_todo;
guint bands; /* number of spectrum bands */
gboolean multi_channel; /* send separate channel results */
guint64 num_frames; /* frame count (1 sample per channel)
* since last emit */
guint64 num_fft; /* number of FFTs since last emit */
GstClockTime message_ts; /* starttime for next message */
/* <private> */
bool channel_data_initialised;
double* input_ring_buffer;
double* fft_input;
fftw_complex* fft_output;
double* spect_magnitude;
fftw_plan plan;
guint input_pos;
guint64 error_per_interval;
guint64 accumulated_error;
GMutex lock;
GstFastSpectrumInputData input_data;
OutputCallback output_callback;
};
struct GstFastSpectrumClass {
GstAudioFilterClass parent_class;
};
GType gst_fastspectrum_get_type (void);
G_END_DECLS
#endif // GST_MOODBAR_FASTSPECTRUM_H_

738
gst/moodbar/gstfftwspectrum.c
View File

@ -1,738 +0,0 @@
/* GStreamer FFTW-based signal-to-spectrum converter
* Copyright (C) 2006 Joseph Rabinoff <bobqwatson@yahoo.com>
*/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
/**
* SECTION:element-fftwspectrum
*
* <refsect2>
* <title>Example launch line</title>
* <para>
* <programlisting>
* gst-launch audiotestsrc ! audioconvert ! fftwspectrum ! fftwunspectrum ! audioconvert ! alsasink
* </programlisting>
* </para>
* </refsect2>
*/
/* This is a simple plugin to take an audio signal and return its
* Fourier transform, using fftw3. It takes a specified number N of
* samples and returns the first N/2+1 (complex) Fourier transform
* values (the other half of the values being the complex conjugates
* of the first). The modulus of these values correspond to the
* strength of the signal in their various bands, and the phase gives
* information about the phase of the signal. The step by which the
* transform increments is also variable, so it can return redundant
* data (to reduce artifacts when converting back into a signal).
*/
#ifdef HAVE_CONFIG_H
# include <config.h>
#endif
#include <gst/gst.h>
#include <fftw3.h>
#include <string.h>
#include <math.h>
#include "gstfftwspectrum.h"
#include "spectrum.h"
GST_DEBUG_CATEGORY (gst_fftwspectrum_debug);
#define GST_CAT_DEFAULT gst_fftwspectrum_debug
/* Filter signals and args */
enum
{
/* FILL ME */
LAST_SIGNAL
};
/* The size and step arguments are actually only default values
* used to fixate the size and step properties of the source cap.
*/
enum
{
ARG_0,
ARG_DEF_SIZE,
ARG_DEF_STEP,
ARG_HIQUALITY
};
#define DEF_SIZE_DEFAULT 1024
#define DEF_STEP_DEFAULT 512
#define HIQUALITY_DEFAULT TRUE
static GstStaticPadTemplate sink_factory
= GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS
( SPECTRUM_SIGNAL_CAPS )
);
/* See spectrum.h for a definition of the frequency caps */
static GstStaticPadTemplate src_factory
= GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS
( SPECTRUM_FREQ_CAPS )
);
G_DEFINE_TYPE(GstFFTWSpectrum, gst_fftwspectrum, GST_TYPE_ELEMENT);
static void gst_fftwspectrum_set_property (GObject *object, guint prop_id,
const GValue *value, GParamSpec *pspec);
static void gst_fftwspectrum_get_property (GObject *object, guint prop_id,
GValue *value, GParamSpec *pspec);
static gboolean gst_fftwspectrum_sink_event(
GstPad* pad, GstObject* parent, GstEvent* event);
static gboolean gst_fftwspectrum_src_event(
GstPad* pad, GstObject* parent, GstEvent* event);
static gboolean gst_fftwspectrum_src_query(
GstPad* pad, GstObject* parent, GstQuery* query);
static gboolean gst_fftwspectrum_sink_query(
GstPad* pad, GstObject* parent, GstQuery* query);
static gboolean gst_fftwspectrum_set_sink_caps (GstPad *pad, GstCaps *caps);
static gboolean gst_fftwspectrum_set_src_caps (GstPad *pad, GstCaps *caps);
static void gst_fftwspectrum_fixatecaps (GstPad *pad, GstCaps *caps);
static GstCaps *gst_fftwspectrum_getcaps (GstPad *pad);
static GstFlowReturn gst_fftwspectrum_chain(
GstPad *pad, GstObject* object, GstBuffer *buf);
static GstStateChangeReturn gst_fftwspectrum_change_state (GstElement *element,
GstStateChange transition);
#define OUTPUT_SIZE(conv) (((conv)->size/2+1)*sizeof(fftw_complex))
/***************************************************************/
/* GObject boilerplate stuff */
/***************************************************************/
/* initialize the plugin's class */
static void
gst_fftwspectrum_class_init (GstFFTWSpectrumClass * klass)
{
GObjectClass *gobject_class;
GstElementClass *gstelement_class;
gobject_class = (GObjectClass *) klass;
gstelement_class = (GstElementClass *) klass;
gobject_class->set_property = gst_fftwspectrum_set_property;
gobject_class->get_property = gst_fftwspectrum_get_property;
g_object_class_install_property (gobject_class, ARG_DEF_SIZE,
g_param_spec_int ("def-size", "Default Size",
"Apply a Fourier transform to this many samples at a time (default value)",
1, G_MAXINT32, DEF_SIZE_DEFAULT, G_PARAM_READWRITE));
g_object_class_install_property (gobject_class, ARG_DEF_STEP,
g_param_spec_int ("def-step", "Default Step",
"Advance the stream this many samples each time (default value)",
1, G_MAXINT32, DEF_STEP_DEFAULT, G_PARAM_READWRITE));
g_object_class_install_property (gobject_class, ARG_HIQUALITY,
g_param_spec_boolean ("hiquality", "High Quality",
"Use a more time-consuming, higher quality algorithm chooser",
HIQUALITY_DEFAULT, G_PARAM_READWRITE));
gstelement_class->change_state
= GST_DEBUG_FUNCPTR (gst_fftwspectrum_change_state);
gst_element_class_add_pad_template (GST_ELEMENT_CLASS(klass),
gst_static_pad_template_get (&src_factory));
gst_element_class_add_pad_template (GST_ELEMENT_CLASS(klass),
gst_static_pad_template_get (&sink_factory));
gst_element_class_set_static_metadata(
GST_ELEMENT_CLASS(klass),
"FFTW-based Fourier transform",
"Filter/Converter/Spectrum",
"Convert a raw audio stream into a frequency spectrum",
"Joe Rabinoff <bobqwatson@yahoo.com>");
g_mutex_init(&klass->mutex);
}
/* initialize the new element
* instantiate pads and add them to element
* set functions
* initialize structure
*/
static void
gst_fftwspectrum_init (GstFFTWSpectrum * conv)
{
GstElementClass* klass =
G_TYPE_INSTANCE_GET_CLASS(conv, GST_ELEMENT_TYPE, GstElementClass);
conv->sinkpad =
gst_pad_new_from_template
(gst_element_class_get_pad_template (klass, "sink"), "sink");
gst_pad_set_event_function(conv->sinkpad, gst_fftwspectrum_sink_event);
gst_pad_set_query_function(conv->srcpad, gst_fftwspectrum_src_query);
gst_pad_set_chain_function (conv->sinkpad,
GST_DEBUG_FUNCPTR (gst_fftwspectrum_chain));
conv->srcpad =
gst_pad_new_from_template
(gst_element_class_get_pad_template (klass, "src"), "src");
gst_pad_set_event_function(conv->srcpad, gst_fftwspectrum_src_event);
gst_pad_set_query_function(conv->sinkpad, gst_fftwspectrum_sink_query);
//gst_pad_set_fixatecaps_function (conv->srcpad,
// GST_DEBUG_FUNCPTR (gst_fftwspectrum_fixatecaps));
gst_element_add_pad (GST_ELEMENT (conv), conv->sinkpad);
gst_element_add_pad (GST_ELEMENT (conv), conv->srcpad);
/* These are set once the (source) capabilities are determined */
conv->rate = 0;
conv->size = 0;
conv->step = 0;
/* These are set when we change to READY */
conv->fftw_in = NULL;
conv->fftw_out = NULL;
conv->fftw_plan = NULL;
/* These are set when we start receiving data */
conv->samples = NULL;
conv->numsamples = 0;
conv->timestamp = 0;
conv->offset = 0;
/* Properties */
conv->def_size = DEF_SIZE_DEFAULT;
conv->def_step = DEF_STEP_DEFAULT;
conv->hi_q = HIQUALITY_DEFAULT;
conv->mutex = &gclass->mutex;
}
static gboolean
gst_fftwspectrum_sink_event(GstPad* pad, GstObject* parent, GstEvent* event)
{
GstFFTWSpectrum* conv = GST_FFTWSPECTRUM(parent);
switch (GST_EVENT_TYPE(event)) {
case GST_EVENT_CAPS: {
GstCaps* caps = NULL;
gst_event_parse_caps(event, &caps);
gst_fftwspectrum_set_sink_caps(pad, caps);
return gst_pad_push_event(conv->srcpad, event);
}
default:
return gst_pad_event_default(pad, parent, event);
}
}
static gboolean
gst_fftwspectrum_src_event(GstPad* pad, GstObject* parent, GstEvent* event)
{
switch (GST_EVENT_TYPE(event)) {
case GST_EVENT_CAPS: {
GstCaps* caps = NULL;
gst_event_parse_caps(event, &caps);
gst_fftwspectrum_set_src_caps(pad, caps);
}
// FALLTHROUGH
default:
return gst_pad_event_default(pad, parent, event);
}
}
static gboolean
gst_fftwspectrum_src_query(GstPad* pad, GstObject* parent, GstQuery* query)
{
switch (GST_QUERY_TYPE(query)) {
case GST_QUERY_CAPS: {
GstCaps* caps = gst_fftwspectrum_getcaps(pad);
gst_pad_set_caps(pad, caps);
}
// FALLTHROUGH
default:
return gst_pad_query_default(pad, parent, query);
}
}
static gboolean
gst_fftwspectrum_sink_query(GstPad* pad, GstObject* parent, GstQuery* query)
{
switch (GST_QUERY_TYPE(query)) {
case GST_QUERY_CAPS: {
GstCaps* caps = gst_fftwspectrum_getcaps(pad);
gst_pad_set_caps(pad, caps);
}
// FALLTHROUGH
default:
return gst_pad_query_default(pad, parent, query);
}
}
static void
gst_fftwspectrum_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstFFTWSpectrum *conv = GST_FFTWSPECTRUM (object);
switch (prop_id)
{
case ARG_DEF_SIZE:
conv->def_size = g_value_get_int (value);
break;
case ARG_DEF_STEP:
conv->def_step = g_value_get_int (value);
break;
case ARG_HIQUALITY:
conv->hi_q = g_value_get_boolean (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_fftwspectrum_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstFFTWSpectrum *conv = GST_FFTWSPECTRUM (object);
switch (prop_id)
{
case ARG_DEF_SIZE:
g_value_set_int (value, conv->def_size);
break;
case ARG_DEF_STEP:
g_value_set_int (value, conv->def_step);
break;
case ARG_HIQUALITY:
g_value_set_boolean (value, conv->hi_q);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
/* Allocate and deallocate fftw state data */
static void
free_fftw_data (GstFFTWSpectrum *conv)
{
if(conv->fftw_plan != NULL)
fftw_destroy_plan (conv->fftw_plan);
if(conv->fftw_in != NULL)
fftw_free (conv->fftw_in);
if(conv->fftw_out != NULL)
fftw_free (conv->fftw_out);
conv->fftw_in = NULL;
conv->fftw_out = NULL;
conv->fftw_plan = NULL;
}
static void
alloc_fftw_data (GstFFTWSpectrum *conv)
{
free_fftw_data (conv);
GST_DEBUG ("Allocating data for size = %d and step = %d",
conv->size, conv->step);
conv->fftw_in = (double *) fftw_malloc (sizeof(double) * conv->size);
conv->fftw_out = (double *) fftw_malloc (OUTPUT_SIZE (conv));
/* We use the simplest real-to-complex algorithm, which takes n real
* inputs and returns floor(n/2) + 1 complex outputs (the other n/2
* outputs are the hermetian conjugates). This should be optimal for
* implementing filters.
*/
g_mutex_lock(conv->mutex);
conv->fftw_plan
= fftw_plan_dft_r2c_1d(conv->size, conv->fftw_in,
(fftw_complex *) conv->fftw_out,
conv->hi_q ? FFTW_MEASURE : FFTW_ESTIMATE);
g_mutex_unlock(conv->mutex);
}
/***************************************************************/
/* Capabilities negotiation */
/***************************************************************/
/* The input and output capabilities are only related by the "rate"
* parameter, which is propagated so that an audio signal can be
* reconstructed eventually. This module does no rate conversion.
*
* The way I understand it, there are two times when caps negotiation
* takes place: (1) when a sink pad receives either its first buffer,
* or a buffer with a new caps type, and (2) when a source pad request
* a buffer from something downstream, and the returned allocated
* buffer has different caps from the ones already negotiated. In the
* first case, _set_sink_caps is called, and in the second, _set_src_caps
* is called.
* When (1) occurs, we remember the rate (the only variable parameter
* in the source) and set the source caps. Then _set_src_caps is called.
* In _set_src_caps, we check that the rate hasn't changed, and figure out
* or remember appropriate size and step attributes. If _set_src_caps is
* called from _set_sink_caps, this completes our setting up our internal
* configuration; if it is called from (2), we reconfigure just the source
* part of the internal configuration.
*/
static gboolean
gst_fftwspectrum_set_sink_caps (GstPad * pad, GstCaps * caps)
{
GstFFTWSpectrum *conv;
GstCaps *srccaps, *newsrccaps;
GstStructure *newstruct;
gint rate;
gboolean res;
conv = GST_FFTWSPECTRUM (gst_pad_get_parent (pad));
srccaps = gst_pad_get_allowed_caps (conv->srcpad);
newsrccaps = gst_caps_copy_nth (srccaps, 0);
gst_caps_unref (srccaps);
newstruct = gst_caps_get_structure (caps, 0);
if (!gst_structure_get_int (newstruct, "rate", &rate))
{
gst_caps_unref (newsrccaps);
gst_object_unref (conv);
return FALSE;
}
/* Fixate the source caps with the given rate */
gst_caps_set_simple (newsrccaps, "rate", G_TYPE_INT, rate, NULL);
//gst_pad_fixate_caps (conv->srcpad, newsrccaps);
conv->rate = rate;
res = gst_pad_set_caps (conv->srcpad, newsrccaps);
if (!res)
conv->rate = 0;
gst_caps_unref (newsrccaps);
gst_object_unref (conv);
return res;
}
static gboolean
gst_fftwspectrum_set_src_caps (GstPad * pad, GstCaps * caps)
{
GstFFTWSpectrum *conv;
gboolean res = FALSE;
GstStructure *newstruct;
gint rate;
conv = GST_FFTWSPECTRUM (gst_pad_get_parent (pad));
newstruct = gst_caps_get_structure (caps, 0);
if (!gst_structure_get_int (newstruct, "rate", &rate))
goto out;
/* Assume caps negotiation has already taken place */
if (rate == conv->rate)
{
gint size, step;
if (!gst_structure_get_int (newstruct, "size", &size))
goto out;
if (!gst_structure_get_int (newstruct, "step", &step))
goto out;
if (conv->size != size || conv->step != step)
{
conv->size = size;
conv->step = step;
/* Re-allocate the fftw data */
if (GST_STATE (GST_ELEMENT (conv)) >= GST_STATE_READY)
alloc_fftw_data (conv);
}
res = TRUE;
}
out:
gst_object_unref (conv);
return res;
}
/* The only thing that can constrain the caps is the rate. */
static GstCaps *
gst_fftwspectrum_getcaps (GstPad *pad)
{
GstFFTWSpectrum *conv;
GstCaps *tmplcaps;
conv = GST_FFTWSPECTRUM (gst_pad_get_parent (pad));
tmplcaps = gst_caps_copy (gst_pad_get_pad_template_caps (pad));
if(conv->rate != 0)
{
/* Assumes the template caps are simple */
gst_caps_set_simple (tmplcaps, "rate", G_TYPE_INT, conv->rate, NULL);
}
gst_object_unref (conv);
return tmplcaps;
}
/* This is called when the source pad needs to choose its capabilities
* when it has a choice and nobody's forcing its hand. In this case
* we take our hint from the def_size and def_step properties.
*/
static void
gst_fftwspectrum_fixatecaps (GstPad *pad, GstCaps *caps)
{
GstFFTWSpectrum *conv;
GstStructure *s;
const GValue *val;
conv = GST_FFTWSPECTRUM (gst_pad_get_parent (pad));
s = gst_caps_get_structure (caps, 0);
val = gst_structure_get_value (s, "size");
if (val == NULL)
gst_caps_set_simple (caps, "size", G_TYPE_INT, conv->def_size, NULL);
else if (G_VALUE_TYPE (val) == GST_TYPE_INT_RANGE)
{
gint sizemin, sizemax;
sizemin = gst_value_get_int_range_min (val);
sizemax = gst_value_get_int_range_max (val);
gst_caps_set_simple (caps, "size", G_TYPE_INT,
CLAMP (conv->def_size, sizemin, sizemax), NULL);
}
/* else it should be already fixed */
val = gst_structure_get_value (s, "step");
if (val == NULL)
gst_caps_set_simple (caps, "step", G_TYPE_INT, conv->def_step, NULL);
else if (G_VALUE_TYPE (val) == GST_TYPE_INT_RANGE)
{
gint stepmin, stepmax;
stepmin = gst_value_get_int_range_min (val);
stepmax = gst_value_get_int_range_max (val);
gst_caps_set_simple (caps, "step", G_TYPE_INT,
CLAMP (conv->def_step, stepmin, stepmax), NULL);
}
/* else it should be already fixed */
/* Assume rate is already fixed (if not it'll be fixed by default) */
gst_object_unref (conv);
}
/***************************************************************/
/* Actual conversion */
/***************************************************************/
static GstStateChangeReturn
gst_fftwspectrum_change_state (GstElement * element,
GstStateChange transition)
{
GstFFTWSpectrum *conv = GST_FFTWSPECTRUM (element);
GstStateChangeReturn res;
switch (transition)
{
case GST_STATE_CHANGE_NULL_TO_READY:
alloc_fftw_data (conv);
break;
case GST_STATE_CHANGE_READY_TO_PAUSED:
conv->samples = (gdouble *) g_malloc (sizeof(gdouble));
conv->numsamples = 0;
conv->timestamp = 0;
conv->offset = 0;
break;
case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
break;
default:
break;
}
res = GST_ELEMENT_CLASS(gst_fftwspectrum_parent_class)
->change_state(element, transition);
switch (transition)
{
case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
break;
case GST_STATE_CHANGE_PAUSED_TO_READY:
g_free(conv->samples);
conv->samples = NULL;
conv->numsamples = 0;
conv->timestamp = 0;
conv->offset = 0;
break;
case GST_STATE_CHANGE_READY_TO_NULL:
free_fftw_data (conv);
break;
default:
break;
}
return res;
}
/* Adds the samples contained in buf to the end of conv->samples,
* updating conv->numsamples.
*/
static void
push_samples (GstFFTWSpectrum *conv, GstBuffer *buf)
{
GstMapInfo map;
gst_buffer_map(buf, &map, GST_MAP_READ);
gint newsamples = map.size / sizeof (gdouble);
gint oldsamples = conv->numsamples;
conv->numsamples += newsamples;
conv->samples = g_realloc (conv->samples, conv->numsamples * sizeof (gdouble));
memcpy (&conv->samples[oldsamples], map.data,
newsamples * sizeof (gdouble));
/* GST_LOG ("Added %d samples", newsamples); */
gst_buffer_unmap(buf, &map);
}
/* This basically does the opposite of push_samples, but takes samples
* off the front.
*/
static void
shift_samples (GstFFTWSpectrum *conv, gint toshift)
{
gdouble *oldsamples = conv->samples;
conv->numsamples -= toshift;
conv->samples = g_malloc (MAX (conv->numsamples, 1) * sizeof (double));
memcpy (conv->samples, &oldsamples[toshift],
conv->numsamples * sizeof (gdouble));
g_free (oldsamples);
/* Fix the timestamp and offset */
conv->timestamp
+= gst_util_uint64_scale_int (GST_SECOND, toshift, conv->rate);
conv->offset += toshift;
/* GST_LOG ("Disposed of %d samples (time: %" GST_TIME_FORMAT " offset: %llu)",
toshift, GST_TIME_ARGS(conv->timestamp), conv->offset); */
}
/* This function queues samples until there are at least
* max (conv->size, conv->step) samples to process. We
* then process samples in chunks of conv->size and increment
* by conv->step.
*/
static GstFlowReturn
gst_fftwspectrum_chain(GstPad* pad, GstObject* object, GstBuffer* buf)
{
GstFFTWSpectrum *conv = GST_FFTWSPECTRUM(object);
GstBuffer *outbuf;
GstFlowReturn res = GST_FLOW_OK;
push_samples (conv, buf);
gst_buffer_unref (buf);
GstQuery* query = gst_query_new_allocation(gst_pad_get_current_caps(pad), TRUE);
if (!gst_pad_peer_query(pad, query)) {
// Query failed, not a problem, we use the query defaults.
}
GstBufferPool* pool = NULL;
guint size = 0;
guint min = 0;
guint max = 0;
if (gst_query_get_n_allocation_pools(query) > 0) {
gst_query_parse_nth_allocation_pool(query, 0, &pool, &size, &min, &max);
}
if (pool == NULL) {
pool = gst_buffer_pool_new();
}
GstStructure* config = gst_buffer_pool_get_config(pool);
gst_buffer_pool_config_set_params(
config, gst_pad_get_current_caps(pad), size, min, max);
gst_buffer_pool_set_config(pool, config);
gst_buffer_pool_set_active(pool, TRUE);
while (conv->numsamples >= MAX (conv->size, conv->step))
{
/*
res = gst_pad_alloc_buffer_and_set_caps(
conv->srcpad,
conv->offset,
OUTPUT_SIZE(conv),
GST_PAD_CAPS(conv->srcpad),
&outbuf);
*/
res = gst_buffer_pool_acquire_buffer(pool, &outbuf, NULL);
if (res != GST_FLOW_OK)
break;
gst_buffer_set_size(outbuf, OUTPUT_SIZE(conv));
GST_BUFFER_OFFSET (outbuf) = conv->offset;
GST_BUFFER_OFFSET_END (outbuf) = conv->offset + conv->step;
GST_BUFFER_TIMESTAMP (outbuf) = conv->timestamp;
GST_BUFFER_DURATION (outbuf) =
gst_util_uint64_scale_int (GST_SECOND, conv->step, conv->rate);
/* Do the Fourier transform */
memcpy (conv->fftw_in, conv->samples, conv->size * sizeof (double));
fftw_execute (conv->fftw_plan);
{ /* Normalize */
gint i;
gfloat root = sqrtf (conv->size);
for (i = 0; i < 2*(conv->size/2+1); ++i) {
conv->fftw_out[i] /= root;
}
}
GstMapInfo map;
gst_buffer_map(outbuf, &map, GST_MAP_WRITE);
memcpy (map.data, conv->fftw_out, OUTPUT_SIZE (conv));
gst_buffer_unmap(outbuf, &map);
res = gst_pad_push (conv->srcpad, outbuf);
shift_samples (conv, conv->step);
if (res != GST_FLOW_OK)
break;
}
gst_object_unref (conv);
return res;
}

72
gst/moodbar/gstfftwspectrum.h
View File

@ -1,72 +0,0 @@
/* GStreamer FFTW-based signal-to-spectrum converter
* Copyright (C) 2006 Joseph Rabinoff <bobqwatson@yahoo.com>
*/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef __GST_FFTWSPECTRUM_H__
#define __GST_FFTWSPECTRUM_H__
#include <gst/gst.h>
#include <fftw3.h>
G_BEGIN_DECLS
/* #defines don't like whitespacey bits */
#define GST_TYPE_FFTWSPECTRUM \
(gst_fftwspectrum_get_type())
#define GST_FFTWSPECTRUM(obj) \
(G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_FFTWSPECTRUM,GstFFTWSpectrum))
#define GST_FFTWSPECTRUM_CLASS(klass) \
(G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_FFTWSPECTRUM,GstFFTWSpectrumClass))
typedef struct _GstFFTWSpectrum GstFFTWSpectrum;
typedef struct _GstFFTWSpectrumClass GstFFTWSpectrumClass;
struct _GstFFTWSpectrum
{
GstElement element;
GstPad *sinkpad, *srcpad;
/* Stream data */
gint rate, size, step;
/* Actual queued (incoming) stream */
gdouble *samples;
gint numsamples;
GstClockTime timestamp; /* Timestamp of the first sample */
guint64 offset; /* Offset of the first sample */
/* State data for fftw */
double *fftw_in;
double *fftw_out;
fftw_plan fftw_plan;
/* Properties */
gint32 def_size, def_step;
gboolean hi_q;
GMutex* mutex;
};
struct _GstFFTWSpectrumClass
{
GstElementClass parent_class;
GMutex mutex;
};
GType gst_fftwspectrum_get_type (void);
G_END_DECLS
#endif /* __GST_FFTWSPECTRUM_H__ */

679
gst/moodbar/gstmoodbar.c
View File

@ -1,679 +0,0 @@
/* GStreamer spectrum analysis toy
* Copyright (C) 2006 Joseph Rabinoff <bobqwatson@yahoo.com>
* Some code copyright (C) 2005 Gav Wood
*/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
/**
* SECTION:element-moodbar
*
* <refsect2>
* <title>Example launch line</title>
* <para>
* <programlisting>
* gst-launch filesrc location=test.mp3 ! mad ! audioconvert ! fftwspectrum ! moodbar height=50 ! pngenc ! filesink location=test.png
* </programlisting>
* </para>
* </refsect2>
*/
/* This plugin is based on the Moodbar code in Amarok version 1.4.0a,
* written by Gav Wood. The algorithm is basically the same as the
* one applied there, and the normalizing code below is taken directly
* from Gav Wood's Exscalibar package.
*/
/* This plugin takes a frequency-domain stream, does some simple
* analysis, and returns a string of (unsigned char) rgb triples
* that represent the magnitude of various sections of the stream.
* Since we have to perform some normalization, we queue up all
* of our analysis until we get an EOS event, at which point we
* normalize and do the output. If a max-width is specified, the
* output is scaled down to the desired width if necessary.
*/
/* More precisely, the analysis performed is as follows:
* (1) the spectrum is broken into 24 parts, called "bark bands"
* (Gav's terminology), as given in bark_bands below
* (2) we compute the size of the first 8 bark bands and store
* that as the "red" component; similarly for blue and green
* (3) after receiving an EOS, we normalize all of the analysis
* done in (1) and (2) and return a stream of rgb triples
* (application/x-raw-rgb)
*/
#ifdef HAVE_CONFIG_H
# include <config.h>
#endif
#include <gst/gst.h>
#include <string.h>
#include <math.h>
#include "gstmoodbar.h"
#include "spectrum.h"
GST_DEBUG_CATEGORY (gst_moodbar_debug);
#define GST_CAT_DEFAULT gst_moodbar_debug
/* Filter signals and args */
enum
{
/* FILL ME */
LAST_SIGNAL
};
enum
{
ARG_0,
ARG_HEIGHT,
ARG_MAX_WIDTH
};
static GstStaticPadTemplate sink_factory
= GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS
( SPECTRUM_FREQ_CAPS )
);
static GstStaticPadTemplate src_factory
= GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS
( "video/x-raw-rgb, "
"bpp = (int) 24, "
"depth = (int) 24, "
"height = (int) [ 1, MAX ], "
"width = (int) [ 1, MAX ], "
"framerate = (fraction) 0/1"
)
);
G_DEFINE_TYPE(GstMoodbar, gst_moodbar, GST_TYPE_ELEMENT);
static void gst_moodbar_set_property (GObject *object, guint prop_id,
const GValue *value, GParamSpec *pspec);
static void gst_moodbar_get_property (GObject *object, guint prop_id,
GValue *value, GParamSpec *pspec);
static gboolean gst_moodbar_set_sink_caps (GstPad *pad, GstCaps *caps);
static gboolean gst_moodbar_sink_event (
GstPad *pad, GstObject *parent, GstEvent *event);
static GstFlowReturn gst_moodbar_chain (
GstPad *pad, GstObject *object, GstBuffer *buf);
static GstStateChangeReturn gst_moodbar_change_state (GstElement *element,
GstStateChange transition);
static void gst_moodbar_finish (GstMoodbar *mood);
/* This is a failsafe so we don't eat up all of a computer's memory
* if we hit an endless stream. */
#define MAX_TRIPLES (1024*1024*4)
#define NUMFREQS(mood) ((mood)->size/2+1)
/* Allocate mood->r, mood->g, and mood->b in chunks of this many */
#define FRAME_CHUNK 1000
/* Default height of the output image */
#define HEIGHT_DEFAULT 1
/* Default max-width of the output image, or 0 for no rescaling */
#define MAX_WIDTH_DEFAULT 0
/* We use this table to break up the incoming spectrum into segments */
static const guint bark_bands[24]
= { 100, 200, 300, 400, 510, 630, 770, 920,
1080, 1270, 1480, 1720, 2000, 2320, 2700, 3150,
3700, 4400, 5300, 6400, 7700, 9500, 12000, 15500 };
/***************************************************************/
/* GObject boilerplate stuff */
/***************************************************************/
/* initialize the plugin's class */
static void
gst_moodbar_class_init (GstMoodbarClass * klass)
{
GObjectClass *gobject_class;
GstElementClass *gstelement_class;
gobject_class = (GObjectClass *) klass;
gstelement_class = (GstElementClass *) klass;
gobject_class->set_property = gst_moodbar_set_property;
gobject_class->get_property = gst_moodbar_get_property;
g_object_class_install_property (gobject_class, ARG_HEIGHT,
g_param_spec_int ("height", "Image height",
"The height of the resulting raw image",
1, G_MAXINT32, HEIGHT_DEFAULT, G_PARAM_READWRITE));
g_object_class_install_property (gobject_class, ARG_MAX_WIDTH,
g_param_spec_int ("max-width", "Image maximum width",
"The maximum width of the resulting raw image, or 0 for no rescaling",
0, G_MAXINT32, MAX_WIDTH_DEFAULT, G_PARAM_READWRITE));
gstelement_class->change_state
= GST_DEBUG_FUNCPTR (gst_moodbar_change_state);
gst_element_class_add_pad_template (gstelement_class,
gst_static_pad_template_get (&src_factory));
gst_element_class_add_pad_template (gstelement_class,
gst_static_pad_template_get (&sink_factory));
gst_element_class_set_static_metadata(
gstelement_class,
"Moodbar analyzer",
"Filter/Converter/Moodbar",
"Convert a spectrum into a stream of (uchar) rgb triples representing its \"mood\"",
"Joe Rabinoff <bobqwatson@yahoo.com>");
}
/* initialize the new element
* instantiate pads and add them to element
* set functions
* initialize structure
*/
static void
gst_moodbar_init (GstMoodbar *mood)
{
GstElementClass *klass = GST_ELEMENT_GET_CLASS (mood);
mood->sinkpad =
gst_pad_new_from_template
(gst_element_class_get_pad_template (klass, "sink"), "sink");
gst_pad_set_event_function (mood->sinkpad,
GST_DEBUG_FUNCPTR (gst_moodbar_sink_event));
gst_pad_set_chain_function (mood->sinkpad,
GST_DEBUG_FUNCPTR (gst_moodbar_chain));
mood->srcpad =
gst_pad_new_from_template
(gst_element_class_get_pad_template (klass, "src"), "src");
gst_element_add_pad (GST_ELEMENT (mood), mood->sinkpad);
gst_element_add_pad (GST_ELEMENT (mood), mood->srcpad);
/* These are set once the (sink) capabilities are determined */
mood->rate = 0;
mood->size = 0;
mood->barkband_table = NULL;
/* These are allocated when we change to PAUSED */
mood->r = NULL;
mood->g = NULL;
mood->b = NULL;
mood->numframes = 0;
/* Property */
mood->height = HEIGHT_DEFAULT;
mood->max_width = MAX_WIDTH_DEFAULT;
}
static void gst_moodbar_set_property (GObject *object, guint prop_id,
const GValue *value, GParamSpec *pspec)
{
GstMoodbar *mood = GST_MOODBAR (object);
switch (prop_id)
{
case ARG_HEIGHT:
mood->height = (guint) g_value_get_int (value);
break;
case ARG_MAX_WIDTH:
mood->max_width = (guint) g_value_get_int (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void gst_moodbar_get_property (GObject *object, guint prop_id,
GValue *value, GParamSpec *pspec)
{
GstMoodbar *mood = GST_MOODBAR (object);
switch (prop_id)
{
case ARG_HEIGHT:
g_value_set_int (value, (int) mood->height);
break;
case ARG_MAX_WIDTH:
g_value_set_int (value, (int) mood->max_width);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
/***************************************************************/
/* Pad handling */
/***************************************************************/
/* This calculates a table that caches which bark band slot each
* incoming band is supposed to go in. */
static void
calc_barkband_table (GstMoodbar *mood)
{
guint i;
guint barkband = 0;
/* Avoid divide-by-zero */
if (!mood->size || !mood->rate)
return;
if (mood->barkband_table)
g_free (mood->barkband_table);
mood->barkband_table = g_malloc (NUMFREQS (mood) * sizeof (guint));
for (i = 0; i < NUMFREQS (mood); ++i)
{
if (barkband < 23 &&
(guint) GST_SPECTRUM_BAND_FREQ (i, mood->size, mood->rate)
>= bark_bands[barkband])
barkband++;
mood->barkband_table[i] = barkband;
/*
GST_LOG ("Band %d (frequency %f) -> barkband %d (frequency %d)",
i, GST_SPECTRUM_BAND_FREQ (i, mood->size, mood->rate),
barkband, bark_bands[barkband]);
*/
}
}
/* Setting the sink caps just gets the rate and size parameters.
* Note that we do not support upstream caps renegotiation, since
* we could only possibly scale the height anyway.
*/
static gboolean
gst_moodbar_set_sink_caps (GstPad *pad, GstCaps *caps)
{
GstMoodbar *mood;
GstStructure *newstruct;
gint rate, size;
gboolean res = FALSE;
mood = GST_MOODBAR (gst_pad_get_parent (pad));
newstruct = gst_caps_get_structure (caps, 0);
if (!gst_structure_get_int (newstruct, "rate", &rate) ||
!gst_structure_get_int (newstruct, "size", &size))
goto out;
res = TRUE;
mood->rate = rate;
mood->size = (guint) size;
calc_barkband_table (mood);
out:
gst_object_unref (mood);
return res;
}
static gboolean
gst_moodbar_sink_event (GstPad *pad, GstObject *parent, GstEvent *event)
{
GstMoodbar *mood = GST_MOODBAR(parent);
gboolean res = TRUE;
if (GST_EVENT_TYPE (event) == GST_EVENT_EOS) {
gst_moodbar_finish (mood);
} else if (GST_EVENT_TYPE(event) == GST_EVENT_CAPS) {
GstCaps* caps = NULL;
gst_event_parse_caps(event, &caps);
gst_moodbar_set_sink_caps(pad, caps);
}
res = gst_pad_push_event (mood->srcpad, event);
gst_object_unref (mood);
return res;
}
/***************************************************************/
/* Actual analysis */
/***************************************************************/
static GstStateChangeReturn
gst_moodbar_change_state (GstElement *element, GstStateChange transition)
{
GstMoodbar *mood = GST_MOODBAR (element);
GstStateChangeReturn res;
switch (transition)
{
case GST_STATE_CHANGE_NULL_TO_READY:
calc_barkband_table (mood);
break;
case GST_STATE_CHANGE_READY_TO_PAUSED:
mood->r = (gdouble *) g_malloc (FRAME_CHUNK * sizeof(gdouble));
mood->g = (gdouble *) g_malloc (FRAME_CHUNK * sizeof(gdouble));
mood->b = (gdouble *) g_malloc (FRAME_CHUNK * sizeof(gdouble));
mood->numframes = 0;
break;
case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
break;
default:
break;
}
res = GST_ELEMENT_CLASS(gst_moodbar_parent_class)
->change_state (element, transition);
switch (transition)
{
case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
break;
case GST_STATE_CHANGE_PAUSED_TO_READY:
g_free (mood->r);
g_free (mood->g);
g_free (mood->b);
mood->r = NULL;
mood->g = NULL;
mood->b = NULL;
mood->numframes = 0;
break;
case GST_STATE_CHANGE_READY_TO_NULL:
g_free (mood->barkband_table);
mood->barkband_table = NULL;
break;
default:
break;
}
return res;
}
/* We allocate r, g, b frames in chunks of FRAME_CHUNK so we don't
* have to realloc every time a buffer comes in.
*/
static gboolean
allocate_another_frame (GstMoodbar *mood)
{
mood->numframes++;
/* Failsafe */
if (mood->numframes == MAX_TRIPLES)
return FALSE;
if(mood->numframes % FRAME_CHUNK == 0)
{
guint size = (mood->numframes + FRAME_CHUNK) * sizeof (gdouble);
mood->r = (gdouble *) g_realloc (mood->r, size);
mood->g = (gdouble *) g_realloc (mood->g, size);
mood->b = (gdouble *) g_realloc (mood->b, size);
if (mood->r == NULL || mood->g == NULL || mood->b == NULL)
return FALSE;
}
return TRUE;
}
/* This function does most of the analysis on the spectra we
* get as input and caches them. We actually push buffers
* once we receive an EOS signal.
*/
static GstFlowReturn
gst_moodbar_chain (GstPad *pad, GstObject *parent, GstBuffer *buf)
{
GstMoodbar *mood = GST_MOODBAR (parent);
guint i;
gdouble amplitudes[24], rgb[3] = {0.f, 0.f, 0.f};
gdouble *out, real, imag;
guint numfreqs = NUMFREQS (mood);
GstMapInfo map;
gst_buffer_map(buf, &map, GST_MAP_READ);
if (map.size != numfreqs * sizeof (gdouble) * 2)
{
gst_buffer_unmap(buf, &map);
gst_object_unref (mood);
return GST_FLOW_ERROR;
}
out = (gdouble *) map.data;
if (!allocate_another_frame (mood))
return GST_FLOW_ERROR;
/* Calculate total amplitudes for the different bark bands */
for (i = 0; i < 24; ++i)
amplitudes[i] = 0.f;
for (i = 0; i < numfreqs; ++i)
{
real = out[2*i]; imag = out[2*i + 1];
amplitudes[mood->barkband_table[i]] += sqrtf (real*real + imag*imag);
}
/* Now divide the bark bands into thirds and compute their total
* amplitudes */
for (i = 0; i < 24; ++i)
rgb[i/8] += amplitudes[i] * amplitudes[i];
rgb[0] = sqrtf (rgb[0]);
rgb[1] = sqrtf (rgb[1]);
rgb[2] = sqrtf (rgb[2]);
mood->r[mood->numframes] = rgb[0];
mood->g[mood->numframes] = rgb[1];
mood->b[mood->numframes] = rgb[2];
gst_buffer_unmap (buf, &map);
gst_buffer_unref (buf);
gst_object_unref (mood);
return GST_FLOW_OK;
}
/* The normalization code was copied from Gav Wood's Exscalibar
* library, normalise.cpp
*/
static void
normalize (gdouble *vals, guint numvals)
{
gdouble mini, maxi, tu = 0.f, tb = 0.f;
gdouble avgu = 0.f, avgb = 0.f, delta, avg = 0.f;
gdouble avguu = 0.f, avgbb = 0.f;
guint i;
gint t = 0;
if (!numvals)
return;
mini = maxi = vals[0];
for (i = 1; i < numvals; i++)
{
if (vals[i] > maxi)
maxi = vals[i];
else if (vals[i] < mini)
mini = vals[i];
}
for (i = 0; i < numvals; i++)
{
if(vals[i] != mini && vals[i] != maxi)
{
avg += vals[i] / ((gdouble) numvals);
t++;
}
}
for (i = 0; i < numvals; i++)
{
if (vals[i] != mini && vals[i] != maxi)
{
if (vals[i] > avg)
{
avgu += vals[i];
tu++;
}
else
{
avgb += vals[i];
tb++;
}
}
}
avgu /= (gdouble) tu;
avgb /= (gdouble) tb;
tu = 0.f;
tb = 0.f;
for (i = 0; i < numvals; i++)
{
if (vals[i] != mini && vals[i] != maxi)
{
if (vals[i] > avgu)
{
avguu += vals[i];
tu++;
}
else if (vals[i] < avgb)
{
avgbb += vals[i];
tb++;
}
}
}
avguu /= (gdouble) tu;
avgbb /= (gdouble) tb;
mini = MAX (avg + (avgb - avg) * 2.f, avgbb);
maxi = MIN (avg + (avgu - avg) * 2.f, avguu);
delta = maxi - mini;
if (delta == 0.f)
delta = 1.f;
for (i = 0; i < numvals; i++)
vals[i] = isfinite (vals[i]) ? MIN(1.f, MAX(0.f, (vals[i] - mini) / delta))
: 0.f;
}
/* This function normalizes all of the cached r,g,b data and
* finally pushes a monster buffer with all of our output.
*/
static void
gst_moodbar_finish (GstMoodbar *mood)
{
GstBuffer *buf;
guchar *data;
guint line;
guint output_width;
if (mood->max_width == 0
|| mood->numframes <= mood->max_width)
output_width = mood->numframes;
else
output_width = mood->max_width;
normalize (mood->r, mood->numframes);
normalize (mood->g, mood->numframes);
normalize (mood->b, mood->numframes);
buf = gst_buffer_new_and_alloc
(output_width * mood->height * 3 * sizeof (guchar));
if (!buf)
return;
GstMapInfo map;
gst_buffer_map(buf, &map, GST_MAP_READ);
/* Don't set the timestamp, duration, etc. since it's irrelevant */
map.memory->offset = 0;
data = (guchar *) map.data;
gdouble r, g, b;
guint i, j, n;
guint start, end;
for (line = 0; line < mood->height; ++line)
{
for (i = 0; i < output_width; ++i)
{
r = 0.f; g = 0.f; b = 0.f;
start = i * mood->numframes / output_width;
end = (i + 1) * mood->numframes / output_width;
if ( start == end )
end = start + 1;
for( j = start; j < end; j++ )
{
r += mood->r[j] * 255.f;
g += mood->g[j] * 255.f;
b += mood->b[j] * 255.f;
}
n = end - start;
*(data++) = (guchar) (r / ((gdouble) n));
*(data++) = (guchar) (g / ((gdouble) n));
*(data++) = (guchar) (b / ((gdouble) n));
}
}
{ /* Now we (finally) know the width of the image we're pushing */
GstCaps *caps = gst_caps_copy (gst_pad_get_current_caps (mood->srcpad));
gboolean res;
gst_caps_set_simple (caps, "width", G_TYPE_INT, output_width, NULL);
gst_caps_set_simple (caps, "height", G_TYPE_INT, mood->height, NULL);
res = gst_pad_set_caps (mood->srcpad, caps);
/*
if (res)
gst_buffer_set_caps (buf, caps);
*/
gst_caps_unref (caps);
if (!res)
return;
}
gst_pad_push (mood->srcpad, buf);
gst_buffer_unmap(buf, &map);
}

63
gst/moodbar/gstmoodbar.h
View File

@ -1,63 +0,0 @@
/* GStreamer spectrum analysis toy
* Copyright (C) 2006 Joseph Rabinoff <bobqwatson@yahoo.com>
* Some code copyright (C) 2005 Gav Wood
*/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef __GST_MOODBAR_H__
#define __GST_MOODBAR_H__
#include <gst/gst.h>
G_BEGIN_DECLS
/* #defines don't like whitespacey bits */
#define GST_TYPE_MOODBAR \
(gst_moodbar_get_type())
#define GST_MOODBAR(obj) \
(G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_MOODBAR,GstMoodbar))
#define GST_MOODBAR_CLASS(klass) \
(G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_MOODBAR,GstMoodbarClass))
typedef struct _GstMoodbar GstMoodbar;
typedef struct _GstMoodbarClass GstMoodbarClass;
struct _GstMoodbar
{
GstElement element;
GstPad *sinkpad, *srcpad;
/* Stream data */
gint rate, size;
/* Cached band -> bark band table */
guint *barkband_table;
/* Queued moodbar data */
gdouble *r, *g, *b;
guint numframes;
/* Property */
guint height;
guint max_width;
};
struct _GstMoodbarClass
{
GstElementClass parent_class;
};
GType gst_moodbar_get_type (void);
G_END_DECLS
#endif /* __GST_MOODBAR_H__ */

48
gst/moodbar/plugin.cpp Normal file
View File

@ -0,0 +1,48 @@
/* This file is part of Clementine.
Copyright 2014, David Sansome <me@davidsansome.com>
Clementine is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Clementine is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Clementine. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gst/gst.h>
#include "gstfastspectrum.h"
#include "plugin.h"
namespace {
static gboolean plugin_init(GstPlugin* plugin) {
if (!gst_element_register(plugin, "fastspectrum",
GST_RANK_NONE, GST_TYPE_FASTSPECTRUM)) {
return FALSE;
}
return TRUE;
}
} // namespace
int gstfastspectrum_register_static() {
return gst_plugin_register_static(
GST_VERSION_MAJOR,
GST_VERSION_MINOR,
"fastspectrum",
"Fast spectrum analyzer for generating Moodbars",
plugin_init,
"0.1",
"GPL",
"FastSpectrum",
"FastSpectrum",
"https://www.clementine-player.org");
}

25
gst/moodbar/plugin.h Normal file
View File

@ -0,0 +1,25 @@
/* This file is part of Clementine.
Copyright 2014, David Sansome <me@davidsansome.com>
Clementine is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Clementine is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Clementine. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef GST_MOODBAR_PLUGIN_H_
#define GST_MOODBAR_PLUGIN_H_
extern "C" {
int gstfastspectrum_register_static();
}
#endif // GST_MOODBAR_PLUGIN_H_

71
gst/moodbar/spectrum.c
View File

@ -1,71 +0,0 @@
/* GStreamer moodbar plugin globals
* Copyright (C) 2006 Joseph Rabinoff <bobqwatson@yahoo.com>
*/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifdef HAVE_CONFIG_H
# include <config.h>
#endif
#include <gst/gst.h>
#include "gstfftwspectrum.h"
#include "gstmoodbar.h"
#include "spectrum.h"
/***************************************************************/
/* Plugin managing */
/***************************************************************/
GST_DEBUG_CATEGORY_EXTERN (gst_fftwspectrum_debug);
GST_DEBUG_CATEGORY_EXTERN (gst_moodbar_debug);
/* entry point to initialize the plug-in
* initialize the plug-in itself
* register the element factories and pad templates
* register the features
*
* exchange the string 'plugin' with your elemnt name
*/
static gboolean
plugin_init (GstPlugin * plugin)
{
if (!gst_element_register (plugin, "fftwspectrum",
GST_RANK_NONE, GST_TYPE_FFTWSPECTRUM))
return FALSE;
if (!gst_element_register (plugin, "moodbar",
GST_RANK_NONE, GST_TYPE_MOODBAR))
return FALSE;
GST_DEBUG_CATEGORY_INIT (gst_fftwspectrum_debug, "fftwspectrum",
0, "FFTW Sample-to-Spectrum Converter Plugin");
GST_DEBUG_CATEGORY_INIT (gst_moodbar_debug, "moodbar",
0, "Moodbar analyzer");
return TRUE;
}
void gstmoodbar_register_static() {
gst_plugin_register_static(
GST_VERSION_MAJOR,
GST_VERSION_MINOR,
"moodbar",
"Frequency analyzer and converter plugin",
plugin_init,
"0.1.2",
"GPL",
"Moodbar",
"Moodbar",
"http://amarok.kde.org/wiki/Moodbar");
}

67
gst/moodbar/spectrum.h
View File

@ -1,67 +0,0 @@
/* GStreamer moodbar plugin globals
* Copyright (C) 2006 Joseph Rabinoff <bobqwatson@yahoo.com>
*/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#ifndef __SPECTRUM_H__
#define __SPECTRUM_H__
/* Since fftwspectrum and fftwunspectrum are supposed to be
* opposites, they'll be using the same caps: */
#define SPECTRUM_SIGNAL_CAPS "audio/x-raw-float, " \
"rate = (int) [ 1, MAX ], " \
"channels = (int) 1, " \
"endianness = (int) BYTE_ORDER, " \
"width = (int) 64, " \
"signed = (boolean) true"
/* audio/x-spectrum-complex-float is an array of complex floats. A
* complex float is just a pair (r, i) of a real float and an
* imaginary float, each with the specified width. The properties
* are as follows:
* rate: the rate of the original signal
* size: the number of signals processed to make the current buffer
* step: the number of signals advanced after the current buffer
* width: the size of the real & imaginary parts of the data
* endianness: ditto
*
* Each audio/x-spectrum-complex-float buffer represents the Fourier
* transform of size samples, and hence _must_ have exactly
* floor(size/2) + 1 complex floats in it; in other words, its
* buffer size must be (floor(size/2) + 1) * 2 * sizeof(gfloat)
*/
#define SPECTRUM_FREQ_CAPS "audio/x-spectrum-complex-float, " \
"rate = (int) [ 1, MAX ], " \
"endianness = (int) BYTE_ORDER, " \
"width = (int) 64, " \
"size = (int) [ 1, MAX ], " \
"step = (int) [ 1, MAX ]"
/* Given a band number from a spectrum made from size audio
* samples at the given rate, return the frequency that band
* corresponds to.
*/
#define GST_SPECTRUM_BAND_FREQ(band, size, rate) \
(((gfloat)(band))*((gfloat)(rate))/((gfloat)(size)))
#ifdef __cplusplus
extern "C" {
#endif
void gstmoodbar_register_static();
#ifdef __cplusplus
}
#endif
#endif /* __SPECTRUM_H__ */

1
src/CMakeLists.txt
View File

@ -1056,6 +1056,7 @@ optional_source(HAVE_LIBMTP
# Moodbar support
optional_source(HAVE_MOODBAR
SOURCES
moodbar/moodbarbuilder.cpp
moodbar/moodbarcontroller.cpp
moodbar/moodbaritemdelegate.cpp
moodbar/moodbarloader.cpp

4
src/engines/gstengine.cpp
View File

@ -49,7 +49,7 @@
#include "core/utilities.h"
#ifdef HAVE_MOODBAR
#include "gst/moodbar/spectrum.h"
#include "gst/moodbar/plugin.h"
#endif
#ifdef HAVE_LIBPULSE
@ -128,7 +128,7 @@ void GstEngine::InitialiseGstreamer() {
gst_init(nullptr, nullptr);
#ifdef HAVE_MOODBAR
gstmoodbar_register_static();
gstfastspectrum_register_static();
#endif
QSet<QString> plugin_names;

191
src/moodbar/moodbarbuilder.cpp Normal file
View File

@ -0,0 +1,191 @@
/* This file is part of Clementine.
Copyright 2014, David Sansome <me@davidsansome.com>
Clementine is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Clementine is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Clementine. If not, see <http://www.gnu.org/licenses/>.
*/
#include "moodbarbuilder.h"
#include "core/arraysize.h"
#include <cmath>
namespace {
static const int sBarkBands[] = {
100, 200, 300, 400, 510, 630, 770, 920,
1080, 1270, 1480, 1720, 2000, 2320, 2700, 3150,
3700, 4400, 5300, 6400, 7700, 9500, 12000, 15500 };
static const int sBarkBandCount = arraysize(sBarkBands);
} // namespace
MoodbarBuilder::MoodbarBuilder()
: bands_(0),
rate_hz_(0) {
}
int MoodbarBuilder::BandFrequency(int band) const {
return ((rate_hz_ / 2) * band + rate_hz_ / 4) / bands_;
}
void MoodbarBuilder::Init(int bands, int rate_hz) {
bands_ = bands;
rate_hz_ = rate_hz;
barkband_table_.clear();
barkband_table_.reserve(bands + 1);
int barkband = 0;
for (int i = 0; i < bands + 1; ++i) {
if (barkband < sBarkBandCount - 1 &&
BandFrequency(i) >= sBarkBands[barkband]) {
barkband++;
}
barkband_table_.append(barkband);
}
}
void MoodbarBuilder::AddFrame(const double* magnitudes, int size) {
if (size > barkband_table_.length()) {
return;
}
// Calculate total magnitudes for different bark bands.
double bands[sBarkBandCount];
for (int i = 0; i < sBarkBandCount; ++i) {
bands[i] = 0.0;
}
for (int i = 0; i < size; ++i) {
bands[barkband_table_[i]] += magnitudes[i];
}
// Now divide the bark bands into thirds and compute their total amplitudes.
double rgb[] = {0, 0, 0};
for (int i = 0; i < sBarkBandCount; ++i) {
rgb[(i * 3) / sBarkBandCount] += bands[i] * bands[i];
}
frames_.append(Rgb(sqrt(rgb[0]), sqrt(rgb[1]), sqrt(rgb[2])));
}
void MoodbarBuilder::Normalize(QList<Rgb>* vals, double Rgb::*member) {
double mini = vals->at(0).*member;
double maxi = vals->at(0).*member;
for (int i = 1; i < vals->count(); i++) {
const double value = vals->at(i).*member;
if (value > maxi) {
maxi = value;
} else if (value < mini) {
mini = value;
}
}
double avg = 0;
int t = 0;
for (const Rgb& rgb : *vals) {
const double value = rgb.*member;
if (value != mini && value != maxi) {
avg += value / vals->count();
t++;
}
}
double tu = 0;
double tb = 0;
double avgu = 0;
double avgb = 0;
for (const Rgb& rgb : *vals) {
const double value = rgb.*member;
if (value != mini && value != maxi) {
if (value > avg) {
avgu += value;
tu++;
} else {
avgb += value;
tb++;
}
}
}
avgu /= tu;
avgb /= tb;
tu = 0;
tb = 0;
double avguu = 0;
double avgbb = 0;
for (const Rgb& rgb : *vals) {
const double value = rgb.*member;
if (value != mini && value != maxi) {
if (value > avgu) {
avguu += value;
tu++;
} else if (value < avgb) {
avgbb += value;
tb++;
}
}
}
avguu /= tu;
avgbb /= tb;
mini = std::max(avg + (avgb - avg) * 2, avgbb);
maxi = std::min(avg + (avgu - avg) * 2, avguu);
double delta = maxi - mini;
if (delta == 0) {
delta = 1;
}
for (auto it = vals->begin(); it != vals->end(); ++it) {
double* value = &((*it).*member);
*value = std::isfinite(*value)
? qBound(0.0, (*value - mini) / delta, 1.0)
: 0;
}
}
QByteArray MoodbarBuilder::Finish(int width) {
Normalize(&frames_, &Rgb::r);
Normalize(&frames_, &Rgb::g);
Normalize(&frames_, &Rgb::b);
QByteArray ret;
ret.resize(width * 3);
char* data = ret.data();
for (int i = 0; i < width; ++i) {
Rgb rgb;
int start = i * frames_.count() / width;
int end = (i + 1) * frames_.count() / width;
if (start == end) {
end = start + 1;
}
for (int j = start; j < end; j++) {
const Rgb& frame = frames_[j];
rgb.r += frame.r * 255;
rgb.g += frame.g * 255;
rgb.b += frame.b * 255;
}
const int n = end - start;
*(data++) = rgb.r / n;
*(data++) = rgb.g / n;
*(data++) = rgb.b / n;
}
return ret;
}

50
src/moodbar/moodbarbuilder.h Normal file
View File

@ -0,0 +1,50 @@
/* This file is part of Clementine.
Copyright 2014, David Sansome <me@davidsansome.com>
Clementine is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Clementine is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Clementine. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef MOODBARBUILDER_H
#define MOODBARBUILDER_H
#include <QColor>
#include <QList>
class MoodbarBuilder {
public:
MoodbarBuilder();
void Init(int bands, int rate_hz);
void AddFrame(const double* magnitudes, int size);
QByteArray Finish(int width);
private:
struct Rgb {
Rgb() : r(0), g(0), b(0) {}
Rgb(double r_, double g_, double b_) : r(r_), g(g_), b(b_) {}
double r, g, b;
};
int BandFrequency(int band) const;
static void Normalize(QList<Rgb>* vals, double Rgb::*member);
QList<uint> barkband_table_;
int bands_;
int rate_hz_;
QList<Rgb> frames_;
};
#endif // MOODBARBUILDER_H

79
src/moodbar/moodbarpipeline.cpp
View File

@ -23,9 +23,14 @@
#include "core/logging.h"
#include "core/signalchecker.h"
#include "core/timeconstants.h"
#include "core/utilities.h"
#include "moodbar/moodbarbuilder.h"
#include "gst/moodbar/gstfastspectrum.h"
bool MoodbarPipeline::sIsAvailable = false;
const int MoodbarPipeline::kBands = 128;
MoodbarPipeline::MoodbarPipeline(const QUrl& local_filename)
: QObject(nullptr),
@ -44,12 +49,6 @@ bool MoodbarPipeline::IsAvailable() {
}
gst_object_unref(factory);
factory = gst_element_factory_find("moodbar");
if (!factory) {
return false;
}
gst_object_unref(factory);
sIsAvailable = true;
}
@ -82,40 +81,44 @@ void MoodbarPipeline::Start() {
GstElement* decodebin = CreateElement("uridecodebin");
convert_element_ = CreateElement("audioconvert");
GstElement* fftwspectrum = CreateElement("fftwspectrum");
GstElement* moodbar = CreateElement("moodbar");
GstElement* appsink = CreateElement("appsink");
GstElement* spectrum = CreateElement("fastspectrum");
GstElement* fakesink = CreateElement("fakesink");
if (!decodebin || !convert_element_ || !fftwspectrum || !moodbar ||
!appsink) {
if (!decodebin || !convert_element_ || !spectrum || !fakesink) {
pipeline_ = nullptr;
emit Finished(false);
return;
}
// Join them together
gst_element_link_many(convert_element_, fftwspectrum, moodbar, appsink,
nullptr);
if (!gst_element_link(convert_element_, spectrum) ||
!gst_element_link(spectrum, fakesink)) {
qLog(Error) << "Failed to link elements";
pipeline_ = nullptr;
emit Finished(false);
return;
}
builder_.reset(new MoodbarBuilder);
// Set properties
g_object_set(decodebin, "uri", local_filename_.toEncoded().constData(),
g_object_set(decodebin,
"uri", local_filename_.toEncoded().constData(),
nullptr);
g_object_set(fftwspectrum, "def-size", 2048, "def-step", 1024, "hiquality",
true, nullptr);
g_object_set(moodbar, "height", 1, "max-width", 1000, nullptr);
g_object_set(spectrum,
"bands", kBands,
nullptr);
GstFastSpectrum* fast_spectrum = GST_FASTSPECTRUM(spectrum);
fast_spectrum->output_callback = [this](double* magnitudes, int size) {
builder_->AddFrame(magnitudes, size);
};
// Connect signals
CHECKED_GCONNECT(decodebin, "pad-added", &NewPadCallback, this);
gst_bus_set_sync_handler(gst_pipeline_get_bus(GST_PIPELINE(pipeline_)),
BusCallbackSync, this, nullptr);
// Set appsink callbacks
GstAppSinkCallbacks callbacks;
memset(&callbacks, 0, sizeof(callbacks));
callbacks.new_sample = NewBufferCallback;
gst_app_sink_set_callbacks(reinterpret_cast<GstAppSink*>(appsink), &callbacks,
this, nullptr);
GstBus* bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline_));
gst_bus_set_sync_handler(bus, BusCallbackSync, this, nullptr);
gst_object_unref(bus);
// Start playing
gst_element_set_state(pipeline_, GST_STATE_PLAYING);
@ -146,21 +149,14 @@ void MoodbarPipeline::NewPadCallback(GstElement*, GstPad* pad, gpointer data) {
gst_pad_link(pad, audiopad);
gst_object_unref(audiopad);
}
GstFlowReturn MoodbarPipeline::NewBufferCallback(GstAppSink* app_sink,
gpointer data) {
MoodbarPipeline* self = reinterpret_cast<MoodbarPipeline*>(data);
int rate = 0;
GstCaps* caps = gst_pad_get_current_caps(pad);
GstStructure* structure = gst_caps_get_structure(caps, 0);
gst_structure_get_int(structure, "rate", &rate);
gst_caps_unref(caps);
GstSample* sample = gst_app_sink_pull_sample(app_sink);
GstBuffer* buffer = gst_sample_get_buffer(sample);
GstMapInfo map;
gst_buffer_map(buffer, &map, GST_MAP_READ);
self->data_.append(reinterpret_cast<const char*>(map.data), map.size);
gst_buffer_unmap(buffer, &map);
gst_buffer_unref(buffer);
return GST_FLOW_OK;
self->builder_->Init(kBands, rate);
}
GstBusSyncReply MoodbarPipeline::BusCallbackSync(GstBus*, GstMessage* msg,
@ -185,6 +181,9 @@ GstBusSyncReply MoodbarPipeline::BusCallbackSync(GstBus*, GstMessage* msg,
void MoodbarPipeline::Stop(bool success) {
success_ = success;
data_ = builder_->Finish(1000);
builder_.reset();
emit Finished(success);
}

9
src/moodbar/moodbarpipeline.h
View File

@ -24,6 +24,10 @@
#include <gst/gst.h>
#include <gst/app/gstappsink.h>
#include <memory>
class MoodbarBuilder;
// Creates moodbar data for a single local music file.
class MoodbarPipeline : public QObject {
Q_OBJECT
@ -40,7 +44,7 @@ class MoodbarPipeline : public QObject {
public slots:
void Start();
signals:
signals:
void Finished(bool success);
private:
@ -58,11 +62,14 @@ signals:
private:
static bool sIsAvailable;
static const int kBands;
QUrl local_filename_;
GstElement* pipeline_;
GstElement* convert_element_;
std::unique_ptr<MoodbarBuilder> builder_;
bool success_;
QByteArray data_;
};