Third party data for songs!

This commit is contained in:
Buster Neece 2014-05-30 08:22:31 -05:00
parent d8cad3ee84
commit d25a6f6821
16 changed files with 857 additions and 43 deletions

View File

@ -21,31 +21,102 @@ class Debug
// Logging
static function log($entry)
{
if (self::$echo_debug)
{
if (DF_IS_COMMAND_LINE)
echo "\n".$entry;
else
echo '<div>'.$entry.'</div>';
}
$row = array('type' => 'log', 'message' => $entry);
self::$debug_log[] = $entry;
if (self::$echo_debug)
self::display($row);
self::$debug_log[] = $row;
}
static function print_r($item)
{
if (DF_IS_COMMAND_LINE)
{
$return_value = print_r($item, TRUE);
}
else
{
$return_value = '<pre style="font-size: 13px; font-family: Consolas, Courier New, Courier, monospace; color: #000; background: #EFEFEF; border: 1px solid #CCC; padding: 5px;">';
$return_value .= print_r($item, TRUE);
$return_value .= '</pre>';
}
$row = array('type' => 'array', 'message' => $item);
self::log($return_value);
if (self::$echo_debug)
self::display($row);
self::$debug_log[] = $row;
}
static function divider()
{
$row = array('type' => 'divider');
if (self::$echo_debug)
self::display($row);
self::$debug_log[] = $row;
}
static function display($info)
{
switch($info['type'])
{
case 'divider':
if (DF_IS_COMMAND_LINE)
{
echo '---------------------------------------------'."\n";
}
else
{
echo '<div style="
padding: 3px;
background: #DDD;
border-left: 4px solid #DDD;
border-bottom: 1px solid #DDD;
margin: 0;"></div>';
}
break;
case 'array':
if (DF_IS_COMMAND_LINE)
{
echo print_r($info['message'], TRUE);
echo "\n";
}
else
{
echo '<pre style="
padding: 3px;
font-family: Consolas, Courier New, Courier, monospace;
font-size: 12px;
background: #EEE;
border-left: 4px solid #FFD24D;
border-bottom: 1px solid #DDD;
margin: 0;">';
$message = print_r($info['message'], TRUE);
if ($message)
echo $message;
else
echo '&nbsp;';
echo '</pre>';
}
break;
case 'log':
default:
if (DF_IS_COMMAND_LINE)
{
echo $info['message']."\n";
}
else
{
echo '<div style="
padding: 3px;
font-family: Consolas, Courier New, Courier, monospace;
font-size: 12px;
background: #EEE;
border-left: 4px solid #4DA6FF;
border-bottom: 1px solid #DDD;
margin: 0;">';
echo $info['message'];
echo '</div>';
}
break;
}
}
// Retrieval
@ -54,18 +125,10 @@ class Debug
return self::$debug_log;
}
static function printLog($preformatted = true)
static function printLog()
{
if ($preformatted)
echo '<pre>';
foreach(self::$debug_log as $log_row)
{
echo $log_row."\n";
}
if ($preformatted)
echo '</pre>';
self::display($log_row);
}
// Timers

View File

@ -11,6 +11,8 @@ use \Entity\Settings;
class NowPlaying
{
static $song_changes = array();
public static function get($version = 1, $id = NULL)
{
$raw_data = @file_get_contents(self::getFilePath());
@ -85,6 +87,16 @@ class NowPlaying
// Post statistics to official record.
Statistic::post($nowplaying);
// Pull external data for newly updated songs.
if (count(self::$song_changes) > 0)
{
foreach(self::$song_changes as $song_id)
{
$song_obj = Song::find($song_id);
$song_obj->syncExternal();
}
}
return $pvl_file_path;
}
@ -238,6 +250,8 @@ class NowPlaying
$np['song_id'] = $song_obj->id;
$np['song_sh_id'] = $sh_obj->id;
$np['song_score'] = SongVote::getScoreForStation($song_obj, $station);
self::$song_changes[] = $np['song_id'];
}
// Get currently active event (cached query)

View File

@ -0,0 +1,70 @@
<?php
namespace PVL\Service;
use \Entity\Song;
use \Entity\SongExternalBronyTunes;
class BronyTunes
{
public static function load()
{
set_time_limit(180);
// Get existing IDs to avoid unnecessary work.
$existing_ids = SongExternalBronyTunes::getIds();
$remote_url = 'https://bronytunes.com/retrieve_songs.php?client_type=ponyvillelive';
$result_raw = @file_get_contents($remote_url);
$em = SongExternalBronyTunes::getEntityManager();
if ($result_raw)
{
$result = json_decode($result_raw, TRUE);
$i = 1;
foreach((array)$result as $row)
{
$id = $row['song_id'];
$processed = SongExternalBronyTunes::processRemote($row);
if (isset($existing_ids[$id]))
{
if ($existing_ids[$id] != $processed['hash'])
{
$record = SongExternalBronyTunes::find($id);
}
else
{
$record = NULL;
}
}
else
{
$record = new SongExternalBronyTunes;
}
if ($record instanceof SongExternalBronyTunes)
{
$record->fromArray($processed);
$em->persist($record);
}
if ($i % 300 == 0)
{
$em->flush();
$em->clear();
}
$i++;
}
$em->flush();
$em->clear();
return true;
}
return false;
}
}

View File

@ -0,0 +1,75 @@
<?php
namespace PVL\Service;
use \Entity\Song;
class EqBeats
{
public static function fetch(Song $song)
{
$result = self::_exactSearch($song);
if (!$result)
$result = self::_querySearch($song);
\PVL\Debug::print_r($result);
if ($result)
return $result;
else
return NULL;
}
protected static function _exactSearch($song)
{
$base_url = 'https://eqbeats.org/tracks/search/exact/json';
$url = $base_url.'?'.http_build_query(array(
'artist' => $song->artist,
'track' => $song->title,
'client' => 'ponyvillelive',
));
\PVL\Debug::log('Exact Search: '.$url);
$result = file_get_contents($url);
if ($result)
{
$rows = json_decode($result, TRUE);
if (count($rows) > 0)
return $rows[0];
}
return NULL;
}
protected static function _querySearch($song)
{
$base_url = 'https://eqbeats.org/tracks/search/json';
$url = $base_url.'?'.http_build_query(array(
'q' => $song->artist.' '.$song->title,
'client' => 'ponyvillelive',
));
\PVL\Debug::log('Query Search: '.$url);
$result = file_get_contents($url);
if ($result)
{
$rows = json_decode($result, TRUE);
foreach($rows as $row)
{
$song_hash = Song::getSongHash(array(
'artist' => $row['user']['name'],
'title' => $row['title'],
));
if (strcmp($song_hash, $song->id) == 0)
return $row;
}
}
return NULL;
}
}

View File

@ -0,0 +1,50 @@
<?php
namespace PVL\Service;
use \Entity\Song;
class PonyFm
{
public static function fetch(Song $song)
{
$base_url = 'https://pony.fm/api/v1/tracks/radio-details/';
$song_hash = self::_getHash($song);
$url = $base_url.$song_hash.'?client=ponyvillelive';
\PVL\Debug::log('Hash Search: '.$url);
$result_raw = @file_get_contents($url);
if ($result_raw)
{
$result = json_decode($result_raw, TRUE);
\PVL\Debug::print_r($result);
return $result;
}
return NULL;
}
protected static function _getHash($song)
{
if ($song->artist)
{
$song_artist = $song->artist;
$song_title = $song->title;
}
else
{
list($song_artist, $song_title) = explode('-', $song->text);
}
return md5(self::_sanitize($song_artist).' - '.self::_sanitize($song_title));
}
protected static function _sanitize($value)
{
$value = preg_replace('/[^A-Za-z0-9]/', '', $value);
return strtolower($value);
}
}

View File

@ -12,6 +12,8 @@ use \Doctrine\Common\Collections\ArrayCollection;
*/
class Song extends \DF\Doctrine\Entity
{
const SYNC_THRESHOLD = 604800; // 604800 = 1 week
public function __construct()
{
$this->created_at = new \DateTime('NOW');
@ -48,6 +50,102 @@ class Song extends \DF\Doctrine\Entity
/** @Column(name="score", type="smallint") */
protected $score;
public function updateScore()
{
$this->score = SongVote::getScore($this);
}
/* External Records */
/** @Column(name="external_timestamp", type="integer", nullable=true) */
protected $external_timestamp;
/** @Column(name="external_ponyfm_id", type="integer", nullable=true) */
protected $external_ponyfm_id;
/**
* @ManyToOne(targetEntity="SongExternalPonyFm")
* @JoinColumns({ @JoinColumn(name="external_ponyfm_id", referencedColumnName="id", onDelete="CASCADE") })
*/
protected $external_ponyfm;
/** @Column(name="external_eqbeats_id", type="integer", nullable=true) */
protected $external_eqbeats_id;
/**
* @ManyToOne(targetEntity="SongExternalEqBeats")
* @JoinColumns({ @JoinColumn(name="external_eqbeats_id", referencedColumnName="id", onDelete="CASCADE") })
*/
protected $external_eqbeats;
/** @Column(name="external_bronytunes_id", type="integer", nullable=true) */
protected $external_bronytunes_id;
/**
* @ManyToOne(targetEntity="SongExternalBronyTunes")
* @JoinColumns({ @JoinColumn(name="external_bronytunes_id", referencedColumnName="id", onDelete="CASCADE") })
*/
protected $external_bronytunes;
public function hasExternal()
{
$adapters = self::getExternalAdapters();
foreach($adapters as $adapter_key => $adapter_class)
{
$local_key = 'external_'.$adapter_key.'_id';
if ($this->{$local_key} !== NULL)
return true;
}
return false;
}
public function getExternal()
{
$adapters = self::getExternalAdapters();
$external = array();
foreach($adapters as $adapter_key => $adapter_class)
{
$local_key = 'external_'.$adapter_key;
if ($this->{$local_key} instanceof $adapter_class)
{
$local_row = $this->{$local_key}->toArray();
unset($local_row['__isInitialized__']);
$external[$adapter_key] = $local_row;
}
}
return $external;
}
public function syncExternal($force = false)
{
$threshold = time()-self::SYNC_THRESHOLD;
if ($this->external_timestamp >= $threshold && !$force)
{
\PVL\Debug::log('Skipping external sync, has been synced recently.');
return false;
}
$adapters = self::getExternalAdapters();
foreach($adapters as $adapter_key => $remote_class)
{
$local_key = 'external_'.$adapter_key;
$this->{$local_key} = $remote_class::match($this, $force);
}
$this->external_timestamp = time();
$this->save();
return true;
}
/* End External Records */
/**
* @OneToMany(targetEntity="SongVote", mappedBy="song")
* @OrderBy({"timestamp" = "DESC"})
@ -60,11 +158,6 @@ class Song extends \DF\Doctrine\Entity
*/
protected $history;
public function updateScore()
{
$this->score = SongVote::getScore($this);
}
/**
* Static Functions
*/
@ -88,7 +181,7 @@ class Song extends \DF\Doctrine\Entity
}
// Generate hash.
if ($song_info['text'])
if (!empty($song_info['text']))
$song_text = $song_info['text'];
else
$song_text = $song_info['artist'].' - '.$song_info['title'];
@ -135,4 +228,13 @@ class Song extends \DF\Doctrine\Entity
'title' => $row['title'],
);
}
public static function getExternalAdapters()
{
return array(
'ponyfm' => '\Entity\SongExternalPonyFm',
'eqbeats' => '\Entity\SongExternalEqBeats',
'bronytunes' => '\Entity\SongExternalBronyTunes',
);
}
}

View File

@ -0,0 +1,102 @@
<?php
namespace Entity;
use \Doctrine\Common\Collections\ArrayCollection;
/**
* @Table(name="song_external_bronytunes", indexes={
* @index(name="search_idx", columns={"hash"}),
* @index(name="sort_idx", columns={"timestamp"}),
* })
* @Entity
*/
class SongExternalBronyTunes extends \DF\Doctrine\Entity
{
/**
* @Column(name="id", type="integer")
* @Id
*/
protected $id;
/** @Column(name="hash", type="string", length=50) */
protected $hash;
/** @Column(name="timestamp", type="integer") */
protected $timestamp;
/** @Column(name="artist", type="string", length=150, nullable=true) */
protected $artist;
/** @Column(name="title", type="string", length=150, nullable=true) */
protected $title;
/** @Column(name="album", type="string", length=150, nullable=true) */
protected $album;
/** @Column(name="description", type="text", nullable=true) */
protected $description;
/** @Column(name="lyrics", type="text", nullable=true) */
protected $lyrics;
/** @Column(name="web_url", type="string", length=255, nullable=true) */
protected $web_url;
/** @Column(name="image_url", type="string", length=255, nullable=true) */
protected $image_url;
/** @Column(name="youtube_url", type="string", length=255, nullable=true) */
protected $youtube_url;
/** @Column(name="purchase_url", type="string", length=255, nullable=true) */
protected $purchase_url;
/**
* Static Functions
*/
public static function match(Song $song, $force_lookup = false)
{
$record = self::getRepository()->findOneBy(array('hash' => $song->id));
if ($record instanceof self && $record->timestamp >= $threshold)
return $record;
else
return NULL;
}
public static function processRemote($result)
{
$song_hash = Song::getSongHash(array(
'artist' => $result['artist_name'],
'title' => $result['name'],
));
return array(
'id' => $result['song_id'],
'hash' => $song_hash,
'timestamp' => time(),
'artist' => $result['artist_name'],
'title' => $result['name'],
'album' => $result['album'],
'description' => $result['description'],
'lyrics' => $result['lyrics'],
'web_url' => 'http://bronytunes.com/songs/'.$result['song_id'],
'image_url' => 'http://bronytunes.com/retrieve_artwork.php?song_id='.$result['song_id'].'&size=256',
'youtube_url' => ($result['youtube_id']) ? 'http://youtu.be/'.$result['youtube_id'] : '',
'purchase_url' => $result['purchase_link'],
);
}
public static function getIds()
{
$em = self::getEntityManager();
$ids_raw = $em->createQuery('SELECT sebt.id, sebt.hash FROM '.__CLASS__.' sebt')->getArrayResult();
$ids = array();
foreach($ids_raw as $row)
$ids[$row['id']] = $row['hash'];
return $ids;
}
}

View File

@ -0,0 +1,98 @@
<?php
namespace Entity;
use \Doctrine\Common\Collections\ArrayCollection;
/**
* @Table(name="song_external_eq_beats", indexes={
* @index(name="search_idx", columns={"hash"}),
* @index(name="sort_idx", columns={"timestamp"}),
* })
* @Entity
*/
class SongExternalEqBeats extends \DF\Doctrine\Entity
{
const SYNC_THRESHOLD = 2592000; // 2592000 = 30 days, 86400 = 1 day
/**
* @Column(name="id", type="integer")
* @Id
*/
protected $id;
/** @Column(name="hash", type="string", length=50) */
protected $hash;
/** @Column(name="timestamp", type="integer") */
protected $timestamp;
/** @Column(name="artist", type="string", length=150, nullable=true) */
protected $artist;
/** @Column(name="title", type="string", length=150, nullable=true) */
protected $title;
/** @Column(name="web_url", type="string", length=255, nullable=true) */
protected $web_url;
/** @Column(name="image_url", type="string", length=255, nullable=true) */
protected $image_url;
/**
* Static Functions
*/
public static function match(Song $song, $force_lookup = false)
{
$threshold = time()-self::SYNC_THRESHOLD;
if (!$force_lookup)
{
$record = self::getRepository()->findOneBy(array('hash' => $song->id));
if ($record instanceof self && $record->timestamp >= $threshold)
return $record;
}
return self::lookUp($song);
}
public static function lookUp(Song $song)
{
$result = \PVL\Service\EqBeats::fetch($song);
if ($result)
{
$record_data = self::processRemote($result);
$record = self::find($record_data['id']);
if (!($record instanceof self))
$record = new self;
$record->fromArray($record_data);
$record->save();
return $record;
}
return NULL;
}
public static function processRemote($result)
{
$song_hash = Song::getSongHash(array(
'artist' => $result['artist']['name'],
'title' => $result['title'],
));
return array(
'id' => $result['id'],
'hash' => $song_hash,
'timestamp' => time(),
'artist' => $result['artist']['name'],
'title' => $result['title'],
'web_url' => $result['link'],
'image_url' => $result['download']['art'],
);
}
}

View File

@ -0,0 +1,114 @@
<?php
namespace Entity;
use \Doctrine\Common\Collections\ArrayCollection;
/**
* @Table(name="song_external_pony_fm", indexes={
* @index(name="search_idx", columns={"hash"}),
* @index(name="sort_idx", columns={"timestamp"}),
* })
* @Entity
*/
class SongExternalPonyFm extends \DF\Doctrine\Entity
{
const SYNC_THRESHOLD = 2592000; // 2592000 = 30 days, 86400 = 1 day
/**
* @Column(name="id", type="integer")
* @Id
*/
protected $id;
/** @Column(name="hash", type="string", length=50) */
protected $hash;
/** @Column(name="timestamp", type="integer") */
protected $timestamp;
/** @Column(name="artist", type="string", length=150, nullable=true) */
protected $artist;
/** @Column(name="title", type="string", length=150, nullable=true) */
protected $title;
/** @Column(name="description", type="text", nullable=true) */
protected $description;
/** @Column(name="lyrics", type="text", nullable=true) */
protected $lyrics;
/** @Column(name="web_url", type="string", length=255, nullable=true) */
protected $web_url;
/** @Column(name="image_url", type="string", length=255, nullable=true) */
protected $image_url;
/** @Column(name="is_vocal", type="boolean") */
protected $is_vocal;
/** @Column(name="is_explicit", type="boolean") */
protected $is_explicit;
/**
* Static Functions
*/
public static function match(Song $song, $force_lookup = false)
{
$threshold = time()-self::SYNC_THRESHOLD;
if (!$force_lookup)
{
$record = self::getRepository()->findOneBy(array('hash' => $song->id));
if ($record instanceof self && $record->timestamp >= $threshold)
return $record;
}
return self::lookUp($song);
}
public static function lookUp(Song $song)
{
$result = \PVL\Service\PonyFm::fetch($song);
if ($result)
{
$record_data = self::processRemote($result);
$record = self::find($record_data['id']);
if (!($record instanceof self))
$record = new self;
$record->fromArray($record_data);
$record->save();
return $record;
}
return NULL;
}
public static function processRemote($result)
{
$song_hash = Song::getSongHash(array(
'artist' => $result['user']['name'],
'title' => $result['title'],
));
return array(
'id' => $result['id'],
'hash' => $song_hash,
'timestamp' => time(),
'artist' => $result['user']['name'],
'title' => $result['title'],
'description' => $result['description'],
'lyrics' => $result['lyrics'],
'web_url' => $result['url'],
'image_url' => $result['covers']['normal'],
'is_vocal' => (int)$result['is_vocal'],
'is_explicit' => (int)$result['is_explicit'],
);
}
}

View File

@ -0,0 +1,37 @@
<?php
use \Entity\Song;
use \Entity\Song as Record;
use \Entity\SongHistory;
use \Entity\SongVote;
class Admin_SongsController extends \DF\Controller\Action
{
public function permissions()
{
return \DF\Acl::getInstance()->isAllowed('administer stations');
}
public function indexAction()
{
}
public function votesAction()
{
$threshold = strtotime('-1 week');
$votes_raw = $this->em->createQuery('SELECT sv.song_id, SUM(sv.vote) AS vote_total FROM Entity\SongVote sv WHERE sv.timestamp >= :threshold GROUP BY sv.song_id')
->setParameter('threshold', $threshold)
->getArrayResult();
\PVL\Utilities::orderBy($votes_raw, 'vote_total DESC');
$votes = array();
foreach($votes_raw as $row)
{
$row['song'] = Song::find($row['song_id']);
$votes[] = $row;
}
$this->view->votes = $votes;
}
}

View File

@ -26,6 +26,9 @@ if ($skin == "dark")
<li><a href="<?=$this->route(array('module' => 'admin', 'controller' => 'rotators')) ?>"><i class="icon-refresh"></i> Rotating Banners</a></li>
<li><a href="<?=$this->route(array('module' => 'admin', 'controller' => 'affiliates')) ?>"><i class="icon-group"></i> Affiliates</a></li>
<li class="nav-header">Network Statistics</li>
<li><a href="<?=$this->route(array('module' => 'admin', 'controller' => 'songs', 'action' => 'votes')) ?>"><i class="icon-sort-by-order"></i> Top Songs for Week</a></li>
<li class="nav-header">Manage System Settings</li>
<li><a href="<?=$this->route(array('module' => 'admin', 'controller' => 'users')) ?>"><i class="icon-user"></i> User Accounts</a></li>
<li><a href="<?=$this->route(array('module' => 'admin', 'controller' => 'permissions')) ?>"><i class="icon-key"></i> Roles &amp; Permissions</a></li>

View File

@ -0,0 +1,41 @@
<?
$this->headTitle('Top Songs from the Last Week');
?>
<table class="datatable table-striped table-condensed table-nopadding">
<colgroup>
<col width="10%">
<col width="90%">
</colgroup>
<thead>
<tr>
<th>Score</th>
<th>Song Title</th>
</tr>
</thead>
<tbody>
<? foreach($this->votes as $song_row): ?>
<tr class="input">
<td class="center">
<big>
<? if ($song_row['vote_total'] > 0): ?>
<span class="text-success"><i class="icon-thumbs-up"></i> <?=$song_row['vote_total'] ?></span>
<? elseif ($song_row['vote_total'] < 0): ?>
<span class="text-error"><i class="icon-thumbs-down"></i> <?=abs($song_row['vote_total']) ?></span>
<? else: ?>
0
<? endif; ?>
</big>
</td>
<td>
<? if ($song_row['song']['title']): ?>
<b><?=$song_row['song']['title'] ?></b><br>
<?=$song_row['song']['artist'] ?>
<? else: ?>
<?=$song_row['song']['text'] ?>
<? endif; ?>
</td>
</tr>
<? endforeach; ?>
</tbody>
</table>

View File

@ -12,12 +12,34 @@ class UtilController extends \DF\Controller\Action
public function testAction()
{
$this->doNotRender();
error_reporting(E_ALL & !E_NOTICE);
echo '<pre>';
\PVL\Debug::showErrors();
\PVL\Debug::setEchoMode(TRUE);
$results = \PVL\NewsAdapter\LiveStream::fetch('http://www.livestream.com/efnwpresents');
\DF\Utilities::print_r($results);
$song = \Entity\Song::find('3618c6feced139030b0306bf15c3fa9c');
$song->syncExternal(true);
$result = $song->getExternal();
\PVL\Debug::print_r($result);
exit;
$np_data = \PVL\NowPlaying::get(2);
$song_ids = array();
foreach($np_data as $station => $station_info)
{
if ($station_info['song_id'])
$song_ids[$station_info['song_id']] = $station_info['text'];
}
foreach($song_ids as $song_id => $text)
{
$song = \Entity\Song::find($song_id);
$song->syncExternal();
$song->save();
\PVL\Debug::divider();
}
exit;
\PVL\NowPlaying::generate();

View File

@ -243,6 +243,12 @@ class Stations_IndexController extends \PVL\Controller\Action\Station
->setParameter('threshold', $threshold)
->getArrayResult();
$ignored_songs = $this->_getIgnoredSongs();
$votes_raw = array_filter($votes_raw, function($value) use ($ignored_songs)
{
return !(isset($ignored_songs[$value['song_id']]));
});
\PVL\Utilities::orderBy($votes_raw, 'vote_total DESC');
$votes = array();

View File

@ -1,6 +1,6 @@
<?php
/**
* Synchronization Script
* Synchronization Script (Runs every 10 minutes).
*/
require_once dirname(__FILE__) . '/../app/bootstrap.php';
@ -23,7 +23,4 @@ ini_set('memory_limit', '256M');
// Sync CentovaCast song data.
\PVL\CentovaCast::sync();
// Sync analytical and statistical data (long running).
\PVL\AnalyticsManager::run();
\Entity\Settings::setSetting('sync_last_run', time());

20
util/syncslow.php Normal file
View File

@ -0,0 +1,20 @@
<?php
/**
* Synchronization Script (Runs every hour).
*/
require_once dirname(__FILE__) . '/../app/bootstrap.php';
$application->bootstrap();
set_time_limit(1800);
error_reporting(E_ALL & ~E_NOTICE);
ini_set('display_errors', 1);
ini_set('memory_limit', '256M');
// Sync the BronyTunes library.
\PVL\Service\BronyTunes::load();
// Sync analytical and statistical data (long running).
\PVL\AnalyticsManager::run();
\Entity\Settings::setSetting('sync_slow_last_run', time());