Funkwhale/strawberry-audio-player-windows-mac-linux
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Make Discogs provider respect rate limiting

Browse Source
This commit is contained in:
Jonas Kvinge 2020-05-14 19:31:40 +02:00
parent e3c367984b
commit 7d4d72e706
2 changed files with 146 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

181
src/covermanager/discogscoverprovider.cpp
View File

@ -32,11 +32,13 @@
#include <QPair> #include <QPair>
#include <QSet> #include <QSet>
#include <QMap> #include <QMap>
#include <QQueue>
#include <QVariant> #include <QVariant>
#include <QString> #include <QString>
#include <QStringList> #include <QStringList>
#include <QUrl> #include <QUrl>
#include <QUrlQuery> #include <QUrlQuery>
#include <QTimer>
#include <QNetworkAccessManager> #include <QNetworkAccessManager>
#include <QNetworkRequest> #include <QNetworkRequest>
#include <QNetworkReply> #include <QNetworkReply>
@ -57,8 +59,17 @@
const char *DiscogsCoverProvider::kUrlSearch = "https://api.discogs.com/database/search"; const char *DiscogsCoverProvider::kUrlSearch = "https://api.discogs.com/database/search";
const char *DiscogsCoverProvider::kAccessKeyB64 = "dGh6ZnljUGJlZ1NEeXBuSFFxSVk="; const char *DiscogsCoverProvider::kAccessKeyB64 = "dGh6ZnljUGJlZ1NEeXBuSFFxSVk=";
const char *DiscogsCoverProvider::kSecretKeyB64 = "ZkFIcmlaSER4aHhRSlF2U3d0bm5ZVmdxeXFLWUl0UXI="; const char *DiscogsCoverProvider::kSecretKeyB64 = "ZkFIcmlaSER4aHhRSlF2U3d0bm5ZVmdxeXFLWUl0UXI=";
const int DiscogsCoverProvider::kRequestsDelay = 1000;
DiscogsCoverProvider::DiscogsCoverProvider(Application *app, QObject *parent) : JsonCoverProvider("Discogs", false, false, 0.0, false, false, app, parent), network_(new NetworkAccessManager(this)) {} DiscogsCoverProvider::DiscogsCoverProvider(Application *app, QObject *parent) :
JsonCoverProvider("Discogs", false, false, 0.0, false, false, app, parent),
network_(new NetworkAccessManager(this)), timer_flush_requests_(new QTimer(this)) {
timer_flush_requests_->setInterval(kRequestsDelay);
timer_flush_requests_->setSingleShot(false);
connect(timer_flush_requests_, SIGNAL(timeout()), this, SLOT(FlushRequests()));
}
DiscogsCoverProvider::~DiscogsCoverProvider() { DiscogsCoverProvider::~DiscogsCoverProvider() {
@ -69,6 +80,9 @@ DiscogsCoverProvider::~DiscogsCoverProvider() {
reply->deleteLater(); reply->deleteLater();
} }
timer_flush_requests_->stop();
queue_search_requests_.clear();
queue_release_requests_.clear();
requests_search_.clear(); requests_search_.clear();
} }
@ -77,23 +91,14 @@ bool DiscogsCoverProvider::StartSearch(const QString &artist, const QString &alb
Q_UNUSED(title); Q_UNUSED(title);
std::shared_ptr<DiscogsCoverSearchContext> search = std::make_shared<DiscogsCoverSearchContext>(); std::shared_ptr<DiscogsCoverSearchContext> search = std::make_shared<DiscogsCoverSearchContext>(id, artist, album);
search->id = id; requests_search_.insert(search->id, search);
search->artist = artist; queue_search_requests_.enqueue(search);
search->album = album;
requests_search_.insert(id, search);
ParamList params = ParamList() << Param("type", "release"); if (!timer_flush_requests_->isActive()) {
if (!search->artist.isEmpty()) { timer_flush_requests_->start();
params.append(Param("artist", search->artist.toLower()));
} }
if (!search->album.isEmpty()) {
params.append(Param("release_title", search->album.toLower()));
}
QNetworkReply *reply = CreateRequest(QUrl(kUrlSearch), params);
connect(reply, &QNetworkReply::finished, [=] { HandleSearchReply(reply, id); });
return true; return true;
@ -105,6 +110,42 @@ void DiscogsCoverProvider::CancelSearch(const int id) {
} }
void DiscogsCoverProvider::FlushRequests() {
if (!queue_release_requests_.isEmpty()) {
SendReleaseRequest(queue_release_requests_.dequeue());
return;
}
if (!queue_search_requests_.isEmpty()) {
SendSearchRequest(queue_search_requests_.dequeue());
return;
}
timer_flush_requests_->stop();
}
void DiscogsCoverProvider::SendSearchRequest(std::shared_ptr<DiscogsCoverSearchContext> search) {
ParamList params = ParamList() << Param("format", "album")
<< Param("artist", search->artist.toLower())
<< Param("release_title", search->album.toLower());
switch (search->type) {
case DiscogsCoverType_Master:
params << Param("type", "master");
break;
case DiscogsCoverType_Release:
params << Param("type", "release");
break;
}
QNetworkReply *reply = CreateRequest(QUrl(kUrlSearch), params);
connect(reply, &QNetworkReply::finished, [=] { HandleSearchReply(reply, search->id); });
}
QNetworkReply *DiscogsCoverProvider::CreateRequest(QUrl url, const ParamList &params_provided) { QNetworkReply *DiscogsCoverProvider::CreateRequest(QUrl url, const ParamList &params_provided) {
ParamList params = ParamList() << Param("key", QByteArray::fromBase64(kAccessKeyB64)) ParamList params = ParamList() << Param("key", QByteArray::fromBase64(kAccessKeyB64))
@ -135,6 +176,8 @@ QNetworkReply *DiscogsCoverProvider::CreateRequest(QUrl url, const ParamList &pa
QNetworkReply *reply = network_->get(req); QNetworkReply *reply = network_->get(req);
replies_ << reply; replies_ << reply;
qLog(Debug) << "Discogs: Sending request" << url;
return reply; return reply;
} }
@ -188,9 +231,7 @@ void DiscogsCoverProvider::HandleSearchReply(QNetworkReply *reply, const int id)
disconnect(reply, nullptr, this, nullptr); disconnect(reply, nullptr, this, nullptr);
reply->deleteLater(); reply->deleteLater();
if (!requests_search_.contains(id)) { if (!requests_search_.contains(id)) return;
return;
}
std::shared_ptr<DiscogsCoverSearchContext> search = requests_search_.value(id); std::shared_ptr<DiscogsCoverSearchContext> search = requests_search_.value(id);
QByteArray data = GetReplyData(reply); QByteArray data = GetReplyData(reply);
@ -221,18 +262,13 @@ void DiscogsCoverProvider::HandleSearchReply(QNetworkReply *reply, const int id)
return; return;
} }
if (!value_results.isArray()) { QJsonArray array_results;
EndSearch(search); if (value_results.isArray()) {
return; array_results = value_results.toArray();
}
QJsonArray array_results = value_results.toArray();
if (array_results.isEmpty()) {
EndSearch(search);
return;
} }
for (const QJsonValue &value_result : array_results) { for (const QJsonValue &value_result : array_results) {
if (!value_result.isObject()) { if (!value_result.isObject()) {
Error("Invalid Json reply, results value is not a object.", value_result); Error("Invalid Json reply, results value is not a object.", value_result);
continue; continue;
@ -244,28 +280,52 @@ void DiscogsCoverProvider::HandleSearchReply(QNetworkReply *reply, const int id)
} }
quint64 release_id = obj_result["id"].toDouble(); quint64 release_id = obj_result["id"].toDouble();
QUrl resource_url(obj_result["resource_url"].toString()); QUrl resource_url(obj_result["resource_url"].toString());
if (!resource_url.isValid()) { QString title = obj_result["title"].toString();
continue;
if (title.contains(" - ")) {
QStringList title_splitted = title.split(" - ");
if (title_splitted.count() == 2) {
QString artist = title_splitted.first();
title = title_splitted.last();
if (artist.toLower() != search->artist.toLower() && title.toLower() != search->album.toLower()) continue;
} }
}
if (!resource_url.isValid()) continue;
if (search->requests_release_.contains(release_id)) { if (search->requests_release_.contains(release_id)) {
continue; continue;
} }
StartRelease(search, release_id, resource_url); StartReleaseRequest(search, release_id, resource_url);
} }
if (search->requests_release_.count() <= 0) { if (search->requests_release_.count() == 0) {
if (search->type == DiscogsCoverType_Master) {
search->type = DiscogsCoverType_Release;
queue_search_requests_.enqueue(search);
}
else {
EndSearch(search); EndSearch(search);
} }
}
} }
void DiscogsCoverProvider::StartRelease(std::shared_ptr<DiscogsCoverSearchContext> search, const quint64 release_id, const QUrl &url) { void DiscogsCoverProvider::StartReleaseRequest(std::shared_ptr<DiscogsCoverSearchContext> search, const quint64 release_id, const QUrl &url) {
DiscogsCoverReleaseContext release(release_id, url); DiscogsCoverReleaseContext release(search->id, release_id, url);
search->requests_release_.insert(release_id, release); search->requests_release_.insert(release_id, release);
queue_release_requests_.enqueue(release);
if (!timer_flush_requests_->isActive()) {
timer_flush_requests_->start();
}
}
void DiscogsCoverProvider::SendReleaseRequest(const DiscogsCoverReleaseContext release) {
QNetworkReply *reply = CreateRequest(release.url); QNetworkReply *reply = CreateRequest(release.url);
connect(reply, &QNetworkReply::finished, [=] { HandleReleaseReply(reply, search->id, release.id); }); connect(reply, &QNetworkReply::finished, [=] { HandleReleaseReply(reply, release.search_id, release.id); });
} }
@ -276,43 +336,39 @@ void DiscogsCoverProvider::HandleReleaseReply(QNetworkReply *reply, const int se
disconnect(reply, nullptr, this, nullptr); disconnect(reply, nullptr, this, nullptr);
reply->deleteLater(); reply->deleteLater();
if (!requests_search_.contains(search_id)) { if (!requests_search_.contains(search_id)) return;
return;
}
std::shared_ptr<DiscogsCoverSearchContext> search = requests_search_.value(search_id); std::shared_ptr<DiscogsCoverSearchContext> search = requests_search_.value(search_id);
if (!search->requests_release_.contains(release_id)) { if (!search->requests_release_.contains(release_id)) return;
return;
}
const DiscogsCoverReleaseContext &release = search->requests_release_.value(release_id); const DiscogsCoverReleaseContext &release = search->requests_release_.value(release_id);
QByteArray data = GetReplyData(reply); QByteArray data = GetReplyData(reply);
if (data.isEmpty()) { if (data.isEmpty()) {
EndSearch(search, release); EndSearch(search, release.id);
return; return;
} }
QJsonObject json_obj = ExtractJsonObj(data); QJsonObject json_obj = ExtractJsonObj(data);
if (json_obj.isEmpty()) { if (json_obj.isEmpty()) {
EndSearch(search, release); EndSearch(search, release.id);
return; return;
} }
if (!json_obj.contains("artists") || !json_obj.contains("title")) { if (!json_obj.contains("artists") || !json_obj.contains("title")) {
Error("Json reply object is missing artists or title.", json_obj); Error("Json reply object is missing artists or title.", json_obj);
EndSearch(search, release); EndSearch(search, release.id);
return; return;
} }
if (!json_obj.contains("images")) { if (!json_obj.contains("images")) {
EndSearch(search, release); EndSearch(search, release.id);
return; return;
} }
QJsonValue value_artists = json_obj["artists"]; QJsonValue value_artists = json_obj["artists"];
if (!value_artists.isArray()) { if (!value_artists.isArray()) {
Error("Json reply object artists is not a array.", value_artists); Error("Json reply object artists is not a array.", value_artists);
EndSearch(search, release); EndSearch(search, release.id);
return; return;
} }
QJsonArray array_artists = value_artists.toArray(); QJsonArray array_artists = value_artists.toArray();
@ -334,28 +390,28 @@ void DiscogsCoverProvider::HandleReleaseReply(QNetworkReply *reply, const int se
} }
if (artist.isEmpty()) { if (artist.isEmpty()) {
EndSearch(search, release); EndSearch(search, release.id);
return; return;
} }
if (i > 1 && artist != search->artist) artist = "Various artists"; if (i > 1 && artist != search->artist) artist = "Various artists";
QString album = json_obj["title"].toString(); QString album = json_obj["title"].toString();
if (artist != search->artist && album != search->album) { if (artist != search->artist && album != search->album) {
EndSearch(search, release); EndSearch(search, release.id);
return; return;
} }
QJsonValue value_images = json_obj["images"]; QJsonValue value_images = json_obj["images"];
if (!value_images.isArray()) { if (!value_images.isArray()) {
Error("Json images is not an array."); Error("Json images is not an array.");
EndSearch(search, release); EndSearch(search, release.id);
return; return;
} }
QJsonArray array_images = value_images.toArray(); QJsonArray array_images = value_images.toArray();
if (array_images.isEmpty()) { if (array_images.isEmpty()) {
Error("Invalid Json reply, images array is empty."); Error("Invalid Json reply, images array is empty.");
EndSearch(search, release); EndSearch(search, release.id);
return; return;
} }
@ -379,28 +435,35 @@ void DiscogsCoverProvider::HandleReleaseReply(QNetworkReply *reply, const int se
if (width < 300 || height < 300) continue; if (width < 300 || height < 300) continue;
const float aspect_score = 1.0 - float(std::max(width, height) - std::min(width, height)) / std::max(height, width); const float aspect_score = 1.0 - float(std::max(width, height) - std::min(width, height)) / std::max(height, width);
if (aspect_score < 0.85) continue; if (aspect_score < 0.85) continue;
CoverSearchResult cover_result; CoverSearchResult result;
cover_result.artist = artist; result.artist = artist;
cover_result.album = album; result.album = album;
cover_result.image_url = QUrl(obj_image["resource_url"].toString()); result.image_url = QUrl(obj_image["resource_url"].toString());
if (cover_result.image_url.isEmpty()) continue; if (result.image_url.isEmpty()) continue;
search->results.append(cover_result); search->results.append(result);
} }
EndSearch(search, release); emit SearchResults(search->id, search->results);
search->results.clear();
EndSearch(search, release.id);
} }
void DiscogsCoverProvider::EndSearch(std::shared_ptr<DiscogsCoverSearchContext> search, const DiscogsCoverReleaseContext &release) { void DiscogsCoverProvider::EndSearch(std::shared_ptr<DiscogsCoverSearchContext> search, const quint64 release_id) {
if (search->requests_release_.contains(release.id)) { if (search->requests_release_.contains(release_id)) {
search->requests_release_.remove(release.id); search->requests_release_.remove(release_id);
} }
if (search->requests_release_.count() <= 0) { if (search->requests_release_.count() <= 0) {
requests_search_.remove(search->id); requests_search_.remove(search->id);
emit SearchFinished(search->id, search->results); emit SearchFinished(search->id, search->results);
} }
if (queue_release_requests_.isEmpty() && queue_search_requests_.isEmpty()) {
timer_flush_requests_->stop();
}
} }
void DiscogsCoverProvider::Error(const QString &error, const QVariant &debug) { void DiscogsCoverProvider::Error(const QString &error, const QVariant &debug) {

32
src/covermanager/discogscoverprovider.h
View File

@ -30,6 +30,7 @@
#include <QMetaType> #include <QMetaType>
#include <QPair> #include <QPair>
#include <QList> #include <QList>
#include <QQueue>
#include <QMap> #include <QMap>
#include <QVariant> #include <QVariant>
#include <QByteArray> #include <QByteArray>
@ -41,6 +42,7 @@
class QNetworkAccessManager; class QNetworkAccessManager;
class QNetworkReply; class QNetworkReply;
class QTimer;
class Application; class Application;
class DiscogsCoverProvider : public JsonCoverProvider { class DiscogsCoverProvider : public JsonCoverProvider {
@ -51,24 +53,25 @@ class DiscogsCoverProvider : public JsonCoverProvider {
~DiscogsCoverProvider(); ~DiscogsCoverProvider();
bool StartSearch(const QString &artist, const QString &album, const QString &title, const int id); bool StartSearch(const QString &artist, const QString &album, const QString &title, const int id);
void CancelSearch(const int id); void CancelSearch(const int id);
private slots: enum DiscogsCoverType {
void HandleSearchReply(QNetworkReply *reply, const int id); DiscogsCoverType_Master,
void HandleReleaseReply(QNetworkReply *reply, const int id, const quint64 release_id); DiscogsCoverType_Release,
};
public:
struct DiscogsCoverReleaseContext { struct DiscogsCoverReleaseContext {
explicit DiscogsCoverReleaseContext(const quint64 _id = 0, const QUrl &_url = QUrl()) : id(_id), url(_url) {} explicit DiscogsCoverReleaseContext(const quint64 _search_id = 0, const quint64 _id = 0, const QUrl &_url = QUrl()) : search_id(_search_id), id(_id), url(_url) {}
quint64 search_id;
quint64 id; quint64 id;
QUrl url; QUrl url;
}; };
struct DiscogsCoverSearchContext { struct DiscogsCoverSearchContext {
explicit DiscogsCoverSearchContext() : id(-1) {} explicit DiscogsCoverSearchContext(const int _id = 0, const QString &_artist = QString(), const QString &_album = QString(), const DiscogsCoverType _type = DiscogsCoverType_Master) : id(_id), artist(_artist), album(_album), type(_type) {}
int id; int id;
QString artist; QString artist;
QString album; QString album;
DiscogsCoverType type;
QMap<quint64, DiscogsCoverReleaseContext> requests_release_; QMap<quint64, DiscogsCoverReleaseContext> requests_release_;
CoverSearchResults results; CoverSearchResults results;
}; };
@ -77,18 +80,29 @@ class DiscogsCoverProvider : public JsonCoverProvider {
typedef QPair<QString, QString> Param; typedef QPair<QString, QString> Param;
typedef QList<Param> ParamList; typedef QList<Param> ParamList;
void SendSearchRequest(std::shared_ptr<DiscogsCoverSearchContext> search);
void SendReleaseRequest(const DiscogsCoverReleaseContext release);
QNetworkReply *CreateRequest(QUrl url, const ParamList &params_provided = ParamList()); QNetworkReply *CreateRequest(QUrl url, const ParamList &params_provided = ParamList());
QByteArray GetReplyData(QNetworkReply *reply); QByteArray GetReplyData(QNetworkReply *reply);
void StartRelease(std::shared_ptr<DiscogsCoverSearchContext> search, const quint64 release_id, const QUrl &url); void StartReleaseRequest(std::shared_ptr<DiscogsCoverSearchContext> search, const quint64 release_id, const QUrl &url);
void EndSearch(std::shared_ptr<DiscogsCoverSearchContext> search, const DiscogsCoverReleaseContext &release = DiscogsCoverReleaseContext()); void EndSearch(std::shared_ptr<DiscogsCoverSearchContext> search, const quint64 release_id = 0);
void Error(const QString &error, const QVariant &debug = QVariant()); void Error(const QString &error, const QVariant &debug = QVariant());
private slots:
void FlushRequests();
void HandleSearchReply(QNetworkReply *reply, const int id);
void HandleReleaseReply(QNetworkReply *reply, const int id, const quint64 release_id);
private: private:
static const char *kUrlSearch; static const char *kUrlSearch;
static const char *kAccessKeyB64; static const char *kAccessKeyB64;
static const char *kSecretKeyB64; static const char *kSecretKeyB64;
static const int kRequestsDelay;
QNetworkAccessManager *network_; QNetworkAccessManager *network_;
QTimer *timer_flush_requests_;
QQueue<std::shared_ptr<DiscogsCoverSearchContext>> queue_search_requests_;
QQueue<DiscogsCoverReleaseContext> queue_release_requests_;
QMap<int, std::shared_ptr<DiscogsCoverSearchContext>> requests_search_; QMap<int, std::shared_ptr<DiscogsCoverSearchContext>> requests_search_;
QList<QNetworkReply*> replies_; QList<QNetworkReply*> replies_;