Implement Influx logging widespread across backend, and on admin view.

This commit is contained in:
Buster Silver 2015-04-19 05:00:02 -05:00
parent 7501c8290b
commit 1a3664c195
9 changed files with 553 additions and 78 deletions

View File

@ -160,6 +160,12 @@ $di->setShared('db', function() use ($config) {
}
});
// InfluxDB
$di->setShared('influx', function() use ($config) {
$opts = $config->influx->toArray();
return new \PVL\Service\InfluxDb($opts);
});
// Auth and ACL
$di->setShared('auth', '\DF\Auth\Model');
$di->setShared('acl', '\DF\Acl\Instance');

View File

@ -27,9 +27,33 @@ class NowPlaying
$nowplaying = self::loadNowPlaying();
// Post statistics to official record.
// Post statistics to official record (legacy for duplication, for now)
Analytics::post($nowplaying['api']);
// Post statistics to InfluxDB.
$influx = self::getInflux();
$influx->setDatabase('pvlive_stations');
$active_shortcodes = Station::getShortNameLookup();
$total_overall = 0;
foreach($nowplaying['api'] as $short_code => $info)
{
$listeners = (int)$info['listeners']['current'];
$station_id = $info['station']['id'];
if (isset($active_shortcodes[$short_code]))
$total_overall += $listeners;
$influx->insert('stations.'.$station_id.'.listeners', [
'value' => $listeners,
]);
}
$influx->insert('all.listeners', [
'value' => $total_overall,
]);
// Clear any records that are not audio/video category.
$api_categories = array('audio', 'video');
foreach($nowplaying['api'] as $station_shortcode => $station_info)
@ -427,4 +451,10 @@ class NowPlaying
$di = \Phalcon\Di::getDefault();
return $di->get('em');
}
public static function getInflux()
{
$di = \Phalcon\Di::getDefault();
return $di->get('influx');
}
}

View File

@ -0,0 +1,199 @@
<?php
namespace PVL\Service;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ParseException;
use GuzzleHttp\Message\ResponseInterface;
class InfluxDb
{
const STATUS_CODE_OK = 200;
const STATUS_CODE_UNAUTHORIZED = 401;
const STATUS_CODE_FORBIDDEN = 403;
const STATUS_CODE_BAD_REQUEST = 400;
/**
* @var array
*/
protected $options;
/**
* @var Client
*/
protected $client;
/**
* @param array $options
*/
public function __construct($options)
{
$defaults = array(
'host' => 'localhost',
'port' => 8086,
'username' => 'root',
'password' => 'root',
'protocol' => 'http',
);
$this->options = array_merge($defaults, (array)$options);
$this->client = new Client();
}
/**
* @param $db_name
*/
public function setDatabase($db_name)
{
$this->options['database'] = $db_name;
return $this;
}
/**
* Insert point into series
* @param string $name
* @param array $value
* @param bool|string $timePrecision
* @return mixed
*/
public function insert($series_name, array $values, $timePrecision = false)
{
$data =[];
$data['name'] = $series_name;
$data['columns'] = array_keys($values);
$data['points'][] = array_values($values);
return $this->rawSend([$data], $timePrecision);
}
/**
* Make a query into database
* @param string $query
* @param bool|string $timePrecision
* @return array
*/
public function query($query, $timePrecision = false)
{
$return = $this->rawQuery($query, $timePrecision);
$response = [];
foreach ($return as $metric) {
$columns = $metric["columns"];
$response[$metric["name"]] = [];
foreach ($metric["points"] as $point) {
$response[$metric["name"]][] = array_combine($columns, $point);
}
}
return $response;
}
/**
* @param $message
* @param bool $timePrecision
* @return \GuzzleHttp\Message\ResponseInterface
*/
public function rawSend($message, $timePrecision = false)
{
$response = $this->client->post(
$this->_getSeriesEndpoint(),
$this->_getRequest($message, [], $timePrecision)
);
return $this->_parseResponse($response);
}
/**
* @param $query
* @param bool $timePrecision
* @return mixed
*/
public function rawQuery($query, $timePrecision = false)
{
$response = $this->client->get(
$this->_getSeriesEndpoint(),
$this->_getRequest([], ["q" => $query], $timePrecision)
);
return $this->_parseResponse($response);
}
/**
* Get the URL for getting/posting updates to a series.
*
* @return string
*/
protected function _getSeriesEndpoint()
{
return sprintf(
"%s://%s:%d/db/%s/series",
$this->options['protocol'],
$this->options['host'],
$this->options['port'],
$this->options['database']
);
}
/**
* @param array $body
* @param array $query
* @param bool $timePrecision
* @return array
*/
protected function _getRequest(array $body = [], array $query = [], $timePrecision = false)
{
$request = [
"auth" => [$this->options['username'], $this->options['password']],
"exceptions" => false
];
if (count($body)) {
$request['body'] = json_encode($body);
}
if (count($query)) {
$request['query'] = $query;
}
if ($timePrecision) {
$request["query"]["time_precision"] = $timePrecision;
}
return $request;
}
/**
* @param ResponseInterface $response
* @return mixed
* @throws \Exception
*/
protected function _parseResponse(ResponseInterface $response)
{
$statusCode = $response->getStatusCode();
if ($statusCode >= 400 && $statusCode < 500) {
$message = (string)$response->getBody();
if (!$message) {
$message = $response->getReasonPhrase();
}
switch ($statusCode) {
case self::STATUS_CODE_UNAUTHORIZED:
case self::STATUS_CODE_FORBIDDEN:
throw new \Exception($message, $statusCode);
case self::STATUS_CODE_BAD_REQUEST:
if (strpos($message, "Couldn't find series:") !== false) {
throw new \Exception($message, $statusCode);
}
}
throw new \Exception($message, $statusCode);
} else if ($statusCode == self::STATUS_CODE_OK) {
try {
return $response->json();
} catch (ParseException $ex) {
throw new \Exception(
sprintf("%s; Response is '%s'", $ex->getMessage(), (string)$response->getBody()),
$ex->getCode(), $ex
);
}
} else if ($statusCode > 200 && $statusCode < 300) {
return true;
}
throw new \Exception((string)$response->getBody(), $statusCode);
}
}

View File

@ -23,44 +23,43 @@ class IndexController extends BaseController
if (!$network_metrics || !$station_metrics) {
// Statistics by day.
$daily_stats = $this->em->createQuery('SELECT a FROM Entity\Analytics a WHERE a.type = :type ORDER BY a.timestamp ASC')
->setParameter('type', 'day')
->getArrayResult();
$influx = $this->di->get('influx');
$influx->setDatabase('pvlive_stations');
$station_averages = array();
$network_data = array(
'PVL Network' => array(
'ranges' => array(),
'averages' => array(),
),
);
foreach ($daily_stats as $stat) {
if (!$stat['station_id']) {
$daily_stats = $influx->query('SELECT * FROM /1d.*/', 'm');
foreach($daily_stats as $stat_series => $stat_rows)
{
$series_split = explode('.', $stat_series);
if ($series_split[1] == 'all')
{
$network_name = 'PVL Network';
$network_data[$network_name]['ranges'][] = array($stat['timestamp'] * 1000, $stat['number_min'], $stat['number_max']);
$network_data[$network_name]['averages'][] = array($stat['timestamp'] * 1000, $stat['number_avg']);
/*} elseif (isset($internal_stations[$stat['station_id']])) {
$network_name = $internal_stations[$stat['station_id']]['name'];
$network_data[$network_name]['ranges'][] = array($stat['timestamp'] * 1000, $stat['number_min'], $stat['number_max']);
$network_data[$network_name]['averages'][] = array($stat['timestamp'] * 1000, $stat['number_avg']);
*/
} else {
$station_averages[$stat['station_id']][] = array($stat['timestamp'] * 1000, $stat['number_avg']);
foreach($stat_rows as $stat_row)
{
$network_data[$network_name]['averages'][] = array($stat_row['time'], $stat_row['value']);
}
}
else
{
$station_id = $series_split[2];
foreach($stat_rows as $stat_row)
{
$station_averages[$station_id][] = array($stat_row['time'], $stat_row['value']);
}
}
}
$network_metrics = array();
foreach ($network_data as $network_name => $data_charts) {
if (isset($data_charts['ranges'])) {
$metric_row = new \stdClass;
$metric_row->name = $network_name . ' Listener Range';
$metric_row->type = 'arearange';
$metric_row->data = $data_charts['ranges'];
$network_metrics[] = $metric_row;
}
if (isset($data_charts['averages'])) {
$metric_row = new \stdClass;
$metric_row->name = $network_name . ' Daily Average';
@ -88,8 +87,8 @@ class IndexController extends BaseController
$network_metrics = json_encode($network_metrics);
$station_metrics = json_encode($station_metrics);
\DF\Cache::save($network_metrics, 'admin_network_metrics', array(), 600);
\DF\Cache::save($station_metrics, 'admin_station_metrics', array(), 600);
// \DF\Cache::save($network_metrics, 'admin_network_metrics', array(), 600);
// \DF\Cache::save($station_metrics, 'admin_station_metrics', array(), 600);
}
$this->view->network_metrics = $network_metrics;

View File

@ -15,10 +15,31 @@ class UtilController extends BaseController
$this->doNotRender();
set_time_limit(0);
ini_set('memory_limit', '-1');
\PVL\Debug::setEchoMode();
\PVL\NotificationManager::run();
$influx = $this->di->get('influx');
$influx->setDatabase('pvlive_stations');
$old_analytics = $this->em->createQuery('SELECT a FROM Entity\Analytics a WHERE a.type = :type')
->setParameter('type', 'day')
->getArrayResult();
foreach($old_analytics as $row)
{
if ($row['station_id'])
$series = 'station.'.$row['station_id'];
else
$series = 'all';
$influx->insert('1d.'.$series.'.listeners', [
'time' => $row['timestamp'],
'value' => $row['number_avg'],
], 's');
}
//\PVL\NotificationManager::run();
\PVL\Debug::log('Donezo!');
}

View File

@ -22,9 +22,10 @@ class IndexController extends BaseController
$threshold = strtotime('-1 month');
// Statistics by day.
$daily_stats = $this->em->createQuery('SELECT a FROM Entity\Analytics a WHERE a.station_id = :station_id AND a.type = :type ORDER BY a.timestamp ASC')
$daily_stats = $this->em->createQuery('SELECT a FROM Entity\Analytics a WHERE a.station_id = :station_id AND a.type = :type AND a.timestamp >= :threshold ORDER BY a.timestamp ASC')
->setParameter('station_id', $this->station->id)
->setParameter('type', 'day')
->setParameter('threshold', $threshold)
->getArrayResult();
$daily_ranges = array();

View File

@ -1,26 +1,27 @@
{
"name": "bravelyblue/pvlive",
"description": "The Ponyville Live! primary application.",
"require": {
"zendframework/zendframework1": "1.12.6",
"doctrine/orm": "2.4.*",
"phpoffice/phpexcel": "1.8.0",
"james-heinrich/getid3": "1.9.8",
"electrolinux/phpquery": "0.9.6",
"google/apiclient": "1.0.*@beta",
"filp/whoops": "1.*"
},
"require-dev": {
"phalcon/devtools": "dev-master"
},
"authors": [
{
"name": "Buster Neece",
"email": "buster@busterneece.com"
}
],
"config": {
"preferred-install": "dist"
},
"minimum-stability": "dev"
"name": "bravelyblue/pvlive",
"description": "The Ponyville Live! primary application.",
"require": {
"zendframework/zendframework1": "1.12.6",
"doctrine/orm": "2.4.*",
"phpoffice/phpexcel": "1.8.0",
"james-heinrich/getid3": "1.9.8",
"electrolinux/phpquery": "0.9.6",
"google/apiclient": "1.0.*@beta",
"filp/whoops": "1.*",
"guzzlehttp/guzzle": "~5.0"
},
"require-dev": {
"phalcon/devtools": "dev-master"
},
"authors": [
{
"name": "Buster Neece",
"email": "buster@busterneece.com"
}
],
"config": {
"preferred-install": "dist"
},
"minimum-stability": "dev"
}

251
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"hash": "6eb598b72d794bf83943f9557fd3abad",
"hash": "6ff1c35a814bb296c33e61434312c98b",
"packages": [
{
"name": "doctrine/annotations",
@ -80,12 +80,12 @@
"source": {
"type": "git",
"url": "https://github.com/doctrine/cache.git",
"reference": "6c5c32eb6c596993d04e13b95d0c1e8153783d7a"
"reference": "c9eadeb743ac6199f7eec423cb9426bc518b7b03"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/doctrine/cache/zipball/6c5c32eb6c596993d04e13b95d0c1e8153783d7a",
"reference": "6c5c32eb6c596993d04e13b95d0c1e8153783d7a",
"url": "https://api.github.com/repos/doctrine/cache/zipball/c9eadeb743ac6199f7eec423cb9426bc518b7b03",
"reference": "c9eadeb743ac6199f7eec423cb9426bc518b7b03",
"shasum": ""
},
"require": {
@ -142,20 +142,20 @@
"cache",
"caching"
],
"time": "2015-02-16 12:24:01"
"time": "2015-04-15 00:11:59"
},
{
"name": "doctrine/collections",
"version": "dev-master",
"version": "v1.3.0",
"source": {
"type": "git",
"url": "https://github.com/doctrine/collections.git",
"reference": "856cb378598f57b3ab6499b1abeb05836feb5725"
"reference": "6c1e4eef75f310ea1b3e30945e9f06e652128b8a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/doctrine/collections/zipball/856cb378598f57b3ab6499b1abeb05836feb5725",
"reference": "856cb378598f57b3ab6499b1abeb05836feb5725",
"url": "https://api.github.com/repos/doctrine/collections/zipball/6c1e4eef75f310ea1b3e30945e9f06e652128b8a",
"reference": "6c1e4eef75f310ea1b3e30945e9f06e652128b8a",
"shasum": ""
},
"require": {
@ -208,20 +208,20 @@
"collections",
"iterator"
],
"time": "2015-04-04 16:54:49"
"time": "2015-04-14 22:21:58"
},
{
"name": "doctrine/common",
"version": "v2.5.0",
"version": "2.5.x-dev",
"source": {
"type": "git",
"url": "https://github.com/doctrine/common.git",
"reference": "cd8daf2501e10c63dced7b8b9b905844316ae9d3"
"reference": "26727ba78de21a824dcbfa5a8ab52c21fe7d71d5"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/doctrine/common/zipball/cd8daf2501e10c63dced7b8b9b905844316ae9d3",
"reference": "cd8daf2501e10c63dced7b8b9b905844316ae9d3",
"url": "https://api.github.com/repos/doctrine/common/zipball/26727ba78de21a824dcbfa5a8ab52c21fe7d71d5",
"reference": "26727ba78de21a824dcbfa5a8ab52c21fe7d71d5",
"shasum": ""
},
"require": {
@ -238,7 +238,7 @@
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "2.6.x-dev"
"dev-master": "2.5.x-dev"
}
},
"autoload": {
@ -281,7 +281,7 @@
"persistence",
"spl"
],
"time": "2015-04-02 19:55:44"
"time": "2015-04-15 22:02:48"
},
{
"name": "doctrine/dbal",
@ -289,12 +289,12 @@
"source": {
"type": "git",
"url": "https://github.com/doctrine/dbal.git",
"reference": "7ff83816f0bf667a97dffc453f5b2181928ffee7"
"reference": "c8c5470edea31f6f47265c78cf907f756a23816a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/doctrine/dbal/zipball/7ff83816f0bf667a97dffc453f5b2181928ffee7",
"reference": "7ff83816f0bf667a97dffc453f5b2181928ffee7",
"url": "https://api.github.com/repos/doctrine/dbal/zipball/c8c5470edea31f6f47265c78cf907f756a23816a",
"reference": "c8c5470edea31f6f47265c78cf907f756a23816a",
"shasum": ""
},
"require": {
@ -353,7 +353,7 @@
"persistence",
"queryobject"
],
"time": "2015-04-09 13:51:10"
"time": "2015-04-15 23:26:40"
},
{
"name": "doctrine/inflector",
@ -692,6 +692,165 @@
],
"time": "2014-09-30 19:33:59"
},
{
"name": "guzzlehttp/guzzle",
"version": "5.2.0",
"source": {
"type": "git",
"url": "https://github.com/guzzle/guzzle.git",
"reference": "475b29ccd411f2fa8a408e64576418728c032cfa"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/guzzle/zipball/475b29ccd411f2fa8a408e64576418728c032cfa",
"reference": "475b29ccd411f2fa8a408e64576418728c032cfa",
"shasum": ""
},
"require": {
"guzzlehttp/ringphp": "~1.0",
"php": ">=5.4.0"
},
"require-dev": {
"ext-curl": "*",
"phpunit/phpunit": "~4.0",
"psr/log": "~1.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "5.0-dev"
}
},
"autoload": {
"psr-4": {
"GuzzleHttp\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Michael Dowling",
"email": "mtdowling@gmail.com",
"homepage": "https://github.com/mtdowling"
}
],
"description": "Guzzle is a PHP HTTP client library and framework for building RESTful web service clients",
"homepage": "http://guzzlephp.org/",
"keywords": [
"client",
"curl",
"framework",
"http",
"http client",
"rest",
"web service"
],
"time": "2015-01-28 01:03:29"
},
{
"name": "guzzlehttp/ringphp",
"version": "dev-master",
"source": {
"type": "git",
"url": "https://github.com/guzzle/RingPHP.git",
"reference": "52d868f13570a9a56e5fce6614e0ec75d0f13ac2"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/RingPHP/zipball/52d868f13570a9a56e5fce6614e0ec75d0f13ac2",
"reference": "52d868f13570a9a56e5fce6614e0ec75d0f13ac2",
"shasum": ""
},
"require": {
"guzzlehttp/streams": "~3.0",
"php": ">=5.4.0",
"react/promise": "~2.0"
},
"require-dev": {
"ext-curl": "*",
"phpunit/phpunit": "~4.0"
},
"suggest": {
"ext-curl": "Guzzle will use specific adapters if cURL is present"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.0-dev"
}
},
"autoload": {
"psr-4": {
"GuzzleHttp\\Ring\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Michael Dowling",
"email": "mtdowling@gmail.com",
"homepage": "https://github.com/mtdowling"
}
],
"description": "Provides a simple API and specification that abstracts away the details of HTTP into a single PHP function.",
"time": "2015-03-30 01:43:20"
},
{
"name": "guzzlehttp/streams",
"version": "dev-master",
"source": {
"type": "git",
"url": "https://github.com/guzzle/streams.git",
"reference": "d1f8a6c55f0f753cfd6f6755856473eb02cedb19"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/streams/zipball/d1f8a6c55f0f753cfd6f6755856473eb02cedb19",
"reference": "d1f8a6c55f0f753cfd6f6755856473eb02cedb19",
"shasum": ""
},
"require": {
"php": ">=5.4.0"
},
"require-dev": {
"phpunit/phpunit": "~4.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "3.0-dev"
}
},
"autoload": {
"psr-4": {
"GuzzleHttp\\Stream\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Michael Dowling",
"email": "mtdowling@gmail.com",
"homepage": "https://github.com/mtdowling"
}
],
"description": "Provides a simple abstraction over streams of data",
"homepage": "http://guzzlephp.org/",
"keywords": [
"Guzzle",
"stream"
],
"time": "2015-01-22 00:01:34"
},
{
"name": "james-heinrich/getid3",
"version": "v1.9.8",
@ -780,6 +939,50 @@
],
"time": "2014-03-02 15:22:49"
},
{
"name": "react/promise",
"version": "v2.2.0",
"source": {
"type": "git",
"url": "https://github.com/reactphp/promise.git",
"reference": "365fcee430dfa4ace1fbc75737ca60ceea7eeeef"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/reactphp/promise/zipball/365fcee430dfa4ace1fbc75737ca60ceea7eeeef",
"reference": "365fcee430dfa4ace1fbc75737ca60ceea7eeeef",
"shasum": ""
},
"require": {
"php": ">=5.4.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "2.0-dev"
}
},
"autoload": {
"psr-4": {
"React\\Promise\\": "src/"
},
"files": [
"src/functions_include.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Jan Sorgalla",
"email": "jsorgalla@googlemail.com"
}
],
"description": "A lightweight implementation of CommonJS Promises/A for PHP",
"time": "2014-12-30 13:32:42"
},
{
"name": "symfony/console",
"version": "2.8.x-dev",
@ -893,12 +1096,12 @@
"source": {
"type": "git",
"url": "https://github.com/phalcon/phalcon-devtools.git",
"reference": "103c8989236f4df5206e7d37508216c46799394a"
"reference": "a7e20addd01c9bc1121cbb68531cd55523320af3"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/phalcon/phalcon-devtools/zipball/103c8989236f4df5206e7d37508216c46799394a",
"reference": "103c8989236f4df5206e7d37508216c46799394a",
"url": "https://api.github.com/repos/phalcon/phalcon-devtools/zipball/a7e20addd01c9bc1121cbb68531cd55523320af3",
"reference": "a7e20addd01c9bc1121cbb68531cd55523320af3",
"shasum": ""
},
"require": {
@ -929,7 +1132,7 @@
"framework",
"phalcon"
],
"time": "2015-02-26 21:47:33"
"time": "2015-04-16 18:51:39"
}
],
"aliases": [],

View File

@ -75,7 +75,7 @@ then
chown -R vagrant /var/log/nginx
unlink /etc/nginx/sites-enabled/default
unlink /etc/nginx/sites-enabled/
# Set up MySQL server.
echo "Customizing MySQL..."
@ -86,6 +86,16 @@ then
echo "CREATE DATABASE pvl CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" | mysql -u root -ppassword
service mysql restart
# Set up InfluxDB
cd ~
wget http://s3.amazonaws.com/influxdb/influxdb_latest_amd64.deb
sudo dpkg -i influxdb_latest_amd64.deb
sudo /etc/init.d/influxdb start
# Preconfigure databases
cd $www_base/util
curl -X POST "http://localhost:8086/cluster/database_configs/pvlive_stations?u=root&p=root" --data-binary @influx_pvlive_stations.json
# Enable PHP flags.
echo "alias phpwww='sudo -u vagrant php'" >> /home/vagrant/.profile
@ -129,6 +139,11 @@ then
cp $www_base/app/config/db.conf.sample.php $www_base/app/config/db.conf.php
fi
if [ ! -f $www_base/app/config/influx.conf.php ]
then
cp $www_base/app/config/influx.conf.sample.php $www_base/app/config/influx.conf.php
fi
# Run Composer.js
if [ ! -f $www_base/vendor/autoload.php ]
then