#2250 -- Handle MaxMind no longer offering free public downloads.

- Remove automatic download attempt from Ansible installation process - Allow listener report to function without the MaxMind database present, but with error messages - Create new "Install GeoLite Database" page in system administration, allowing user upload of the MaxMind database, with instructions
2019-12-31 06:49:25 -06:00 · 2019-12-31 06:49:25 -06:00 · 388bcee951
parent 6290812a2a
commit 388bcee951
17 changed files with 599 additions and 369 deletions
--- a/config/forms/install_geolite.php
+++ b/config/forms/install_geolite.php
@ -0,0 +1,64 @@
+<?php
+return [
+    'method' => 'post',
+    'enctype' => 'multipart/form-data',
+
+    'groups' => [
+        [
+            'use_grid' => true,
+            'elements' => [
+                'details' => [
+                    'markup',
+                    [
+                        'label' => __('Instructions'),
+                        'markup' =>
+                            '<p>' . __('You can upload the MaxMind GeoLite database in order to provide geolocation of the IP addresses of your listeners. This will allow you to view the listeners on each station\'s "Listeners" report. To download the GeoLite database:') . '</p>' .
+                            '<ul>' .
+                            '<li>' . __('Create an account on <a href="%s" target="_blank">the MaxMind developer site</a>.',
+                                'https://www.maxmind.com/en/geolite2/signup') . '</li>' .
+                            '<li>' . __('Visit the <a href="%s" target="_blank">direct downloads page</a>.',
+                                'https://www.maxmind.com/en/download_files?direct=1') . '</li>' .
+                            '<li>' . __('Download the <code>%s</code> file (in GZIP) format.',
+                                'GeoLite2-City') . '</li>' .
+                            '<li>' . __('Select the downloaded file below to upload it.') . '</li>'
+                            . '</ul>' .
+                            '<p>' . __('You can repeat this process any time you need to update the GeoLite database.') . '</p>',
+                        'form_group_class' => 'col-sm-12',
+                    ],
+                ],
+
+                'current_version' => [
+                    'markup',
+                    [
+                        'label' => __('Current Installed Version'),
+                        'markup' => '<p class="text-danger">' . __('GeoLite is not currently installed on this installation.') . '</p>',
+                        'form_group_class' => 'col-sm-12',
+                    ],
+                ],
+
+                'binary' => [
+                    'file',
+                    [
+                        'label' => __('Select GeoLite2-City .tar.gz File'),
+                        'required' => true,
+                        'type' => 'archive',
+                        'max_size' => 50 * 1024 * 1024,
+                        'form_group_class' => 'col-md-6',
+                        'button_text' => __('Select File'),
+                        'button_icon' => 'cloud_upload',
+                    ],
+                ],
+
+                'submit' => [
+                    'submit',
+                    [
+                        'type' => 'submit',
+                        'label' => __('Upload'),
+                        'class' => 'ui-button btn-lg btn-primary',
+                        'form_group_class' => 'col-sm-12',
+                    ],
+                ],
+            ],
+        ],
+    ],
+];
--- a/config/forms/install_shoutcast.php
+++ b/config/forms/install_shoutcast.php
@ -10,19 +10,20 @@ return [
                'details' => [
                    'markup',
                    [
-                        'label' => __('Important Notes'),
-                        'markup' => __('<p>SHOUTcast 2 DNAS is not free software, and its restrictive license does not allow AzuraCast to distribute the SHOUTcast binary. In order to install SHOUTcast, you should download the Linux x64 binary from the <a href="%s" target="_blank">SHOUTcast Radio Manager</a> web site. Upload the <code>sc_serv2_linux_x64-latest.tar.gz</code> into the field below to automatically extract it into the proper directory.</p>', 'https://radiomanager.shoutcast.com/register/serverSoftwareFreemium'),
+                        'label' => __('Instructions'),
+                        'markup' => __('<p>SHOUTcast 2 DNAS is not free software, and its restrictive license does not allow AzuraCast to distribute the SHOUTcast binary. In order to install SHOUTcast, you should download the Linux x64 binary from the <a href="%s" target="_blank">SHOUTcast Radio Manager</a> web site. Upload the <code>sc_serv2_linux_x64-latest.tar.gz</code> into the field below to automatically extract it into the proper directory.</p>',
+                            'https://radiomanager.shoutcast.com/register/serverSoftwareFreemium'),
                        'form_group_class' => 'col-sm-12',
-                    ]
+                    ],
                ],

                'current_version' => [
                    'markup',
                    [
                        'label' => __('Current Installed Version'),
-                        'markup' => '<p class="text-danger">'.__('SHOUTcast is not currently installed on this installation.').'</p>',
+                        'markup' => '<p class="text-danger">' . __('SHOUTcast is not currently installed on this installation.') . '</p>',
                        'form_group_class' => 'col-sm-12',
-                    ]
+                    ],
                ],

                'binary' => [
@ -34,7 +35,7 @@ return [
                        'form_group_class' => 'col-md-6',
                        'button_text' => __('Select File'),
                        'button_icon' => 'cloud_upload',
-                    ]
+                    ],
                ],

                'submit' => [
@ -44,9 +45,9 @@ return [
                        'label' => __('Upload'),
                        'class' => 'ui-button btn-lg btn-primary',
                        'form_group_class' => 'col-sm-12',
-                    ]
+                    ],
                ],
-            ]
-        ]
+            ],
+        ],
    ],
 ];
--- a/config/menus/admin.php
+++ b/config/menus/admin.php
@ -5,7 +5,7 @@

 use App\Acl;

-return function(\App\Event\BuildAdminMenu $e) {
+return function (\App\Event\BuildAdminMenu $e) {
    $router = $e->getRouter();

    $e->merge([
@ -42,7 +42,7 @@ return function(\App\Event\BuildAdminMenu $e) {
                    'label' => __('Backups'),
                    'url' => $router->named('admin:backups:index'),
                    'permission' => Acl::GLOBAL_BACKUPS,
-                ]
+                ],
            ],
        ],
        'users' => [
@ -85,7 +85,12 @@ return function(\App\Event\BuildAdminMenu $e) {
                    'url' => $router->named('admin:install:shoutcast'),
                    'permission' => Acl::GLOBAL_ALL,
                ],
+                'geolite' => [
+                    'label' => __('Install GeoLite IP Database'),
+                    'url' => $router->named('admin:install:geolite'),
+                    'permission' => Acl::GLOBAL_ALL,
+                ],
            ],
-        ]
+        ],
    ]);
 };
--- a/config/routes.php
+++ b/config/routes.php
@ -23,6 +23,9 @@ return function (App $app) {
            $group->map(['GET', 'POST'], '/shoutcast', Controller\Admin\InstallShoutcastController::class)
                ->setName('admin:install:shoutcast');

+            $group->map(['GET', 'POST'], '/geolite', Controller\Admin\InstallGeoLiteController::class)
+                ->setName('admin:install:geolite');
+
        })->add(new Middleware\Permissions(Acl::GLOBAL_ALL));

        $group->get('/auditlog', Controller\Admin\AuditLogController::class)
--- a/config/services.php
+++ b/config/services.php
@ -179,10 +179,7 @@ return [
    },

    // MaxMind (IP Geolocation database for listener metadata)
-    MaxMind\Db\Reader::class => function (Settings $settings) {
-        $mmdb_path = dirname($settings[Settings::BASE_DIR]) . '/geoip/GeoLite2-City.mmdb';
-        return new MaxMind\Db\Reader($mmdb_path);
-    },
+    App\Service\GeoLite::class => DI\autowire(),

    // InfluxDB
    InfluxDB\Database::class => function (Settings $settings) {
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@ -37,6 +37,7 @@ services:
      - tmp_data:/var/azuracast/www_tmp
      - station_data:/var/azuracast/stations
      - shoutcast2_install:/var/azuracast/servers/shoutcast2
+      - geolite_install:/var/azuracast/geoip
      - backups:/var/azuracast/backups
    restart: always
    ulimits: &default-ulimits
@ -141,6 +142,7 @@ volumes:
  influx_data: {}
  station_data: {}
  shoutcast2_install: {}
+  geolite_install: {}
  tmp_data: {}
  redis_data: {}
  backups: {}
--- a/docker-compose.sample.yml
+++ b/docker-compose.sample.yml
@ -38,6 +38,7 @@ services:
      - tmp_data:/var/azuracast/www_tmp
      - station_data:/var/azuracast/stations
      - shoutcast2_install:/var/azuracast/servers/shoutcast2
+      - geolite_install:/var/azuracast/geoip
      - backups:/var/azuracast/backups
    restart: always
    ulimits: &default-ulimits
@ -245,6 +246,7 @@ volumes:
  influx_data: {}
  letsencrypt: {}
  shoutcast2_install: {}
+  geolite_install: {}
  station_data: {}
  tmp_data: {}
  www_data: {}
--- a/resources/locale/default.pot
+++ b/resources/locale/default.pot
--- a/src/Controller/Admin/InstallGeoLiteController.php
+++ b/src/Controller/Admin/InstallGeoLiteController.php
@ -0,0 +1,81 @@
+<?php
+namespace App\Controller\Admin;
+
+use App\Form\Form;
+use App\Http\Response;
+use App\Http\ServerRequest;
+use App\Service\GeoLite;
+use Azura\Config;
+use Exception;
+use Psr\Http\Message\ResponseInterface;
+use Psr\Http\Message\UploadedFileInterface;
+use Symfony\Component\Process\Process;
+use const UPLOAD_ERR_OK;
+
+class InstallGeoLiteController
+{
+    protected array $form_config;
+
+    protected GeoLite $geoLite;
+
+    public function __construct(Config $config, GeoLite $geoLite)
+    {
+        $this->form_config = $config->get('forms/install_geolite');
+        $this->geoLite = $geoLite;
+    }
+
+    public function __invoke(ServerRequest $request, Response $response): ResponseInterface
+    {
+        $form_config = $this->form_config;
+
+        $version = $this->geoLite->getVersion();
+        if (null !== $version) {
+            $form_config['groups'][0]['elements']['current_version'][1]['markup'] = '<p class="text-success">' . __('GeoLite version "%s" is currently installed.',
+                    $version) . '</p>';
+        }
+
+        $form = new Form($form_config, []);
+
+        if ($request->isPost() && $form->isValid($request->getParsedBody())) {
+            try {
+                $baseDir = dirname($this->geoLite->getDatabasePath());
+
+                $files = $request->getUploadedFiles();
+                /** @var UploadedFileInterface $import_file */
+                $import_file = $files['binary'];
+
+                if (UPLOAD_ERR_OK === $import_file->getError()) {
+                    $tgzPath = $baseDir . '/maxmind.tar.gz';
+                    if (file_exists($tgzPath)) {
+                        unlink($tgzPath);
+                    }
+
+                    $import_file->moveTo($tgzPath);
+
+                    $process = new Process([
+                        'tar',
+                        'xvzf',
+                        $tgzPath,
+                        '--strip-components=1',
+                    ], $baseDir);
+
+                    $process->mustRun();
+
+                    unlink($tgzPath);
+                }
+
+                return $response->withRedirect($request->getUri()->getPath());
+            } catch (Exception $e) {
+                $form
+                    ->getField('binary')
+                    ->addError(get_class($e) . ': ' . $e->getMessage());
+            }
+        }
+
+        return $request->getView()->renderToResponse($response, 'system/form_page', [
+            'form' => $form,
+            'render_mode' => 'edit',
+            'title' => __('Install GeoLite IP Database'),
+        ]);
+    }
+}
--- a/src/Controller/Admin/InstallShoutcastController.php
+++ b/src/Controller/Admin/InstallShoutcastController.php
@ -35,7 +35,7 @@ class InstallShoutcastController

        $form = new Form($form_config, []);

-        if ('POST' === $request->getMethod() && $form->isValid($_POST)) {
+        if ($request->isPost() && $form->isValid($request->getParsedBody())) {
            try {
                $sc_base_dir = Settings::getInstance()->getParentDirectory() . '/servers/shoutcast2';

@ -43,7 +43,7 @@ class InstallShoutcastController
                /** @var UploadedFileInterface $import_file */
                $import_file = $files['binary'];

-                if ($import_file->getError() === UPLOAD_ERR_OK) {
+                if (UPLOAD_ERR_OK === $import_file->getError()) {
                    $sc_tgz_path = $sc_base_dir . '/sc_serv.tar.gz';
                    if (file_exists($sc_tgz_path)) {
                        unlink($sc_tgz_path);
@ -58,6 +58,8 @@ class InstallShoutcastController
                    ], $sc_base_dir);

                    $process->mustRun();
+
+                    unlink($sc_tgz_path);
                }

                return $response->withRedirect($request->getUri()->getPath());
--- a/src/Controller/Api/Stations/ListenersController.php
+++ b/src/Controller/Api/Stations/ListenersController.php
@ -4,12 +4,11 @@ namespace App\Controller\Api\Stations;
 use App\Entity;
 use App\Http\Response;
 use App\Http\ServerRequest;
+use App\Service\GeoLite;
 use Azura\Utilities\Csv;
 use Cake\Chronos\Chronos;
 use DateTimeZone;
 use Doctrine\ORM\EntityManager;
-use Exception;
-use MaxMind\Db\Reader;
 use Mobile_Detect;
 use OpenApi\Annotations as OA;
 use Psr\Http\Message\ResponseInterface;
@ -18,12 +17,12 @@ class ListenersController
 {
    protected EntityManager $em;

-    protected Reader $geoip;
+    protected GeoLite $geoLite;

-    public function __construct(EntityManager $em, Reader $geoip)
+    public function __construct(EntityManager $em, GeoLite $geoLite)
    {
        $this->em = $em;
-        $this->geoip = $geoip;
+        $this->geoLite = $geoLite;
    }

    /**
@ -147,7 +146,7 @@ class ListenersController
            ];

            foreach ($listeners_raw as $listener) {
-                $location = $this->_getLocationInfo($listener['listener_ip'], $locale);
+                $location = $this->geoLite->getLocationInfo($listener['listener_ip'], $locale);

                $export_row = [
                    (string)$listener['listener_ip'],
@ -185,60 +184,11 @@ class ListenersController
            $api->is_mobile = $detect->isMobile($listener['listener_user_agent']);
            $api->connected_on = (int)$listener['timestamp_start'];
            $api->connected_time = Entity\Listener::getListenerSeconds($listener['intervals']);
-            $api->location = $this->_getLocationInfo($listener['listener_ip'], $locale);
+            $api->location = $this->geoLite->getLocationInfo($listener['listener_ip'], $locale);

            $listeners[] = $api;
        }

        return $response->withJson($listeners);
    }
-
-    protected function _getLocationInfo($ip, $locale): array
-    {
-        try {
-            $ip_info = $this->geoip->get($ip);
-        } catch (Exception $e) {
-            return [
-                'status' => 'error',
-                'message' => $e->getMessage(),
-            ];
-        }
-
-        if (empty($ip_info)) {
-            return [
-                'status' => 'error',
-                'message' => 'Internal/Reserved IP',
-            ];
-        }
-
-        return [
-            'status' => 'success',
-            'lat' => $ip_info['location']['latitude'] ?? 0.0,
-            'lon' => $ip_info['location']['longitude'] ?? 0.0,
-            'timezone' => $ip_info['location']['time_zone'] ?? '',
-            'region' => $this->_getLocalizedString($ip_info['subdivisions'][0]['names'] ?? null, $locale),
-            'country' => $this->_getLocalizedString($ip_info['country']['names'] ?? null, $locale),
-            'city' => $this->_getLocalizedString($ip_info['city']['names'] ?? null, $locale),
-            'message' => 'This product includes GeoLite2 data created by MaxMind, available from <a href="http://www.maxmind.com">http://www.maxmind.com</a>.',
-        ];
-    }
-
-    protected function _getLocalizedString($names, $locale): string
-    {
-        if (empty($names)) {
-            return '';
-        }
-
-        // Convert "en_US" to "en-US", the format MaxMind uses.
-        $locale = str_replace('_', '-', $locale);
-
-        // Check for an exact match.
-        if (isset($names[$locale])) {
-            return $names[$locale];
-        }
-
-        // Check for a match of the first portion, i.e. "en"
-        $locale = strtolower(substr($locale, 0, 2));
-        return $names[$locale] ?? $names['en'];
-    }
 }
--- a/src/Service/GeoLite.php
+++ b/src/Service/GeoLite.php
@ -0,0 +1,96 @@
+<?php
+namespace App\Service;
+
+use App\Settings;
+use Cake\Chronos\Chronos;
+use MaxMind\Db\Reader;
+
+class GeoLite
+{
+    protected Settings $settings;
+
+    protected ?Reader $reader = null;
+
+    public function __construct(Settings $settings)
+    {
+        $this->settings = $settings;
+
+        $mmdbPath = $this->getDatabasePath();
+        if (file_exists($mmdbPath)) {
+            $this->reader = new Reader($mmdbPath);
+        }
+    }
+
+    public function getDatabasePath(): string
+    {
+        return dirname($this->settings[Settings::BASE_DIR]) . '/geoip/GeoLite2-City.mmdb';
+    }
+
+    public function getVersion(): ?string
+    {
+        if (null === $this->reader) {
+            return null;
+        }
+
+        $metadata = $this->reader->metadata();
+
+        $buildDate = Chronos::createFromTimestampUTC($metadata->buildEpoch);
+        return $metadata->databaseType . ' (' . $buildDate->format('Y-m-d') . ')';
+    }
+
+    public function getLocationInfo(string $ip, string $locale): array
+    {
+        if (null === $this->reader) {
+            return [
+                'status' => 'error',
+                'message' => 'GeoLite database not configured for this installation. See System Administration for instructions.',
+            ];
+        }
+
+        try {
+            $ipInfo = $this->reader->get($ip);
+        } catch (\Exception $e) {
+            return [
+                'status' => 'error',
+                'message' => $e->getMessage(),
+            ];
+        }
+
+        if (empty($ipInfo)) {
+            return [
+                'status' => 'error',
+                'message' => 'Internal/Reserved IP',
+            ];
+        }
+
+        return [
+            'status' => 'success',
+            'lat' => $ipInfo['location']['latitude'] ?? 0.0,
+            'lon' => $ipInfo['location']['longitude'] ?? 0.0,
+            'timezone' => $ipInfo['location']['time_zone'] ?? '',
+            'region' => $this->getLocalizedString($ipInfo['subdivisions'][0]['names'] ?? null, $locale),
+            'country' => $this->getLocalizedString($ipInfo['country']['names'] ?? null, $locale),
+            'city' => $this->getLocalizedString($ipInfo['city']['names'] ?? null, $locale),
+            'message' => 'This product includes GeoLite2 data created by MaxMind, available from <a href="http://www.maxmind.com">http://www.maxmind.com</a>.',
+        ];
+    }
+
+    protected function getLocalizedString($names, string $locale): string
+    {
+        if (empty($names)) {
+            return '';
+        }
+
+        // Convert "en_US" to "en-US", the format MaxMind uses.
+        $locale = str_replace('_', '-', $locale);
+
+        // Check for an exact match.
+        if (isset($names[$locale])) {
+            return $names[$locale];
+        }
+
+        // Check for a match of the first portion, i.e. "en"
+        $locale = strtolower(substr($locale, 0, 2));
+        return $names[$locale] ?? $names['en'];
+    }
+}
--- a/update.sh
+++ b/update.sh
@ -32,7 +32,7 @@ else
 fi

 APP_ENV="${APP_ENV:-production}"
-UPDATE_REVISION="${UPDATE_REVISION:-46}"
+UPDATE_REVISION="${UPDATE_REVISION:-47}"

 echo "Updating AzuraCast (Environment: $APP_ENV, Update revision: $UPDATE_REVISION)"

--- a/util/ansible/deploy.yml
+++ b/util/ansible/deploy.yml
@ -24,7 +24,6 @@
    - mariadb
    - azuracast-db-install
    - ufw
-    - maxmind
    - composer
    - influxdb
    - services
--- a/util/ansible/roles/maxmind/tasks/main.yml
+++ b/util/ansible/roles/maxmind/tasks/main.yml
@ -1,16 +0,0 @@
---
-  - name: Download MaxMind GeoIP Database
-    get_url:
-      url: http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz
-      dest: "{{ app_base }}/geoip/maxmind.tar.gz"
-
-  - name: Extract MaxMind GeoIP Database
-    unarchive:
-      src: "{{ app_base }}/geoip/maxmind.tar.gz"
-      dest: "{{ app_base }}/geoip"
-      remote_src: yes
-      creates: "{{ app_base }}/geoip/GeoLite2-City.mmdb"
-      mode: "u=rwx,g=rx,o=rx"
-      owner: "azuracast"
-      group: "www-data"
-      extra_opts: "--strip-components=1"
--- a/util/ansible/roles/php/tasks/main.yml
+++ b/util/ansible/roles/php/tasks/main.yml
@ -74,7 +74,7 @@
    with_items:
      - { option: 'post_max_size',       value: "50M" }
      - { option: 'short_open_tag',      value: "On" }
-      - { option: 'upload_max_filesize', value: "25M" }
+      - { option: 'upload_max_filesize', value: "50M" }

  - name: Configure php-cli php.ini
    ini_file:
--- a/util/ansible/update.yml
+++ b/util/ansible/update.yml
@ -18,11 +18,10 @@
    - { role: mariadb, when: update_revision|int < 15 }
    - { role: nginx, when: update_revision|int < 44 }
    - { role: redis, when: update_revision|int < 14 }
-    - { role: php, when: update_revision|int < 46 }
+    - { role: php, when: update_revision|int < 47 }
    - composer
    - { role: influxdb, when: update_revision|int < 10 }
    - { role: ufw, when: update_revision|int < 12 }
-    - { role: maxmind, when: update_revision|int < 24 }
    - { role: services, when: update_revision|int < 13 }
    - { role: azuracast-cron, when: update_revision|int < 41 }
    - azuracast-setup