Read tags from Ubuntu One files and add to local database.

This commit is contained in:
John Maguire 2012-11-28 14:43:03 +01:00
parent 45b2c1dcf4
commit db586ca00e
17 changed files with 217 additions and 30 deletions

View File

@ -332,6 +332,7 @@
<file>schema/schema-38.sql</file>
<file>schema/schema-39.sql</file>
<file>schema/schema-3.sql</file>
<file>schema/schema-40.sql</file>
<file>schema/schema-4.sql</file>
<file>schema/schema-5.sql</file>
<file>schema/schema-6.sql</file>

48
data/schema/schema-40.sql Normal file
View File

@ -0,0 +1,48 @@
CREATE TABLE ubuntu_one_songs(
title TEXT,
album TEXT,
artist TEXT,
albumartist TEXT,
composer TEXT,
track INTEGER,
disc INTEGER,
bpm REAL,
year INTEGER,
genre TEXT,
comment TEXT,
compilation INTEGER,
length INTEGER,
bitrate INTEGER,
samplerate INTEGER,
directory INTEGER NOT NULL,
filename TEXT NOT NULL,
mtime INTEGER NOT NULL,
ctime INTEGER NOT NULL,
filesize INTEGER NOT NULL,
sampler INTEGER NOT NULL DEFAULT 0,
art_automatic TEXT,
art_manual TEXT,
filetype INTEGER NOT NULL DEFAULT 0,
playcount INTEGER NOT NULL DEFAULT 0,
lastplayed INTEGER,
rating INTEGER,
forced_compilation_on INTEGER NOT NULL DEFAULT 0,
forced_compilation_off INTEGER NOT NULL DEFAULT 0,
effective_compilation NOT NULL DEFAULT 0,
skipcount INTEGER NOT NULL DEFAULT 0,
score INTEGER NOT NULL DEFAULT 0,
beginning INTEGER NOT NULL DEFAULT 0,
cue_path TEXT,
unavailable INTEGER DEFAULT 0,
effective_albumartist TEXT,
etag TEXT
);
CREATE VIRTUAL TABLE ubuntu_one_songs_fts USING fts3 (
ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment,
tokenize=unicode
);
UPDATE schema_version SET version=40;

View File

@ -6,6 +6,8 @@ include_directories(${CMAKE_BINARY_DIR}/ext/libclementine-tagreader)
include_directories(${CMAKE_SOURCE_DIR}/src)
include_directories(${CMAKE_BINARY_DIR}/src)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++0x -U__STRICT_ANSI__")
set(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR})
set(SOURCES
@ -15,6 +17,7 @@ set(SOURCES
)
set(HEADERS
googledrivestream.h
)
optional_source(HAVE_GOOGLE_DRIVE
@ -24,10 +27,12 @@ optional_source(HAVE_GOOGLE_DRIVE
)
qt4_wrap_cpp(MOC ${HEADERS})
qt4_add_resources(QRC data/data.qrc)
add_executable(clementine-tagreader
${SOURCES}
${MOC}
${QRC}
)
target_link_libraries(clementine-tagreader

View File

@ -0,0 +1,5 @@
<RCC>
<qresource prefix="/certs">
<file>godaddy-root.pem</file>
</qresource>
</RCC>

View File

@ -0,0 +1,24 @@
-----BEGIN CERTIFICATE-----
MIIEADCCAuigAwIBAgIBADANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEh
MB8GA1UEChMYVGhlIEdvIERhZGR5IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBE
YWRkeSBDbGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA0MDYyOTE3
MDYyMFoXDTM0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRo
ZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3Mg
MiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggEN
ADCCAQgCggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCA
PVYYYwhv2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6w
wdhFJ2+qN1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXi
EqITLdiOr18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMY
avx4A6lNf4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+
YihfukEHU1jPEX44dMX4/7VpkI+EdOqXG68CAQOjgcAwgb0wHQYDVR0OBBYEFNLE
sNKR1EwRcbNhyz2h/t2oatTjMIGNBgNVHSMEgYUwgYKAFNLEsNKR1EwRcbNhyz2h
/t2oatTjoWekZTBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMYVGhlIEdvIERhZGR5
IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRpZmlj
YXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQAD
ggEBADJL87LKPpH8EsahB4yOd6AzBhRckB4Y9wimPQoZ+YeAEW5p5JYXMP80kWNy
OO7MHAGjHZQopDH2esRU1/blMVgDoszOYtuURXO1v0XJJLXVggKtI3lpjbi2Tc7P
TMozI+gciKqdi0FuFskg5YmezTvacPd+mSYgFFQlq25zheabIZ0KbIIOqPjCDPoQ
HmyW74cNxA9hi63ugyuV+I6ShHI56yDqg+2DzZduCLzrTia2cyvk0/ZM/iZx4mER
dEr/VxqHD3VILs9RaRegAhJhldXRQLIQTO7ErBBDpqWeCtWVYpoNz4iCxTIM5Cuf
ReYNnyicsbkqWletNw+vHX/bvZ8=
-----END CERTIFICATE-----

View File

@ -109,12 +109,19 @@ TagLib::ByteVector GoogleDriveStream::readBlock(ulong length) {
}
QNetworkRequest request = QNetworkRequest(url_);
request.setRawHeader(
"Authorization", QString("Bearer %1").arg(auth_).toUtf8());
request.setRawHeader("Authorization", auth_.toUtf8());
request.setRawHeader(
"Range", QString("bytes=%1-%2").arg(start).arg(end).toUtf8());
request.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
QNetworkRequest::AlwaysNetwork);
// The Ubuntu One server applies the byte range to the gzipped data, rather
// than the raw data so we must disable compression.
if (url_.host() == "files.one.ubuntu.com") {
request.setRawHeader("Accept-Encoding", "identity");
}
QNetworkReply* reply = network_->get(request);
connect(reply, SIGNAL(sslErrors(QList<QSslError>)), SLOT(SSLErrors(QList<QSslError>)));
++num_requests_;
QEventLoop loop;
@ -183,3 +190,10 @@ long GoogleDriveStream::length() {
void GoogleDriveStream::truncate(long) {
qLog(Debug) << Q_FUNC_INFO << "not implemented";
}
void GoogleDriveStream::SSLErrors(const QList<QSslError>& errors) {
for (const QSslError& error : errors) {
qLog(Debug) << error.error() << error.errorString();
qLog(Debug) << error.certificate();
}
}

View File

@ -1,16 +1,16 @@
/* This file is part of Clementine.
Copyright 2012, David Sansome <me@davidsansome.com>
Clementine is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Clementine is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Clementine. If not, see <http://www.gnu.org/licenses/>.
*/
@ -18,6 +18,9 @@
#ifndef GOOGLEDRIVESTREAM_H
#define GOOGLEDRIVESTREAM_H
#include <QObject>
#include <QList>
#include <QSslError>
#include <QUrl>
#include <google/sparsetable>
@ -25,7 +28,8 @@
class QNetworkAccessManager;
class GoogleDriveStream : public TagLib::IOStream {
class GoogleDriveStream : public QObject, public TagLib::IOStream {
Q_OBJECT
public:
GoogleDriveStream(const QUrl& url,
const QString& filename,
@ -63,6 +67,9 @@ class GoogleDriveStream : public TagLib::IOStream {
void FillCache(int start, TagLib::ByteVector data);
TagLib::ByteVector GetCached(int start, int end);
private slots:
void SSLErrors(const QList<QSslError>& errors);
private:
const QUrl url_;
const QString filename_;

View File

@ -21,6 +21,7 @@
#include <QCoreApplication>
#include <QLocalSocket>
#include <QSslSocket>
#include <QStringList>
#include <iostream>
@ -57,6 +58,9 @@ int main(int argc, char** argv) {
return 1;
}
QSslSocket::addDefaultCaCertificates(
QSslCertificate::fromPath(":/certs/godaddy-root.pem", QSsl::Pem));
TagReaderWorker worker(&socket);
return a.exec();

View File

@ -139,7 +139,7 @@ void TagReaderWorker::MessageArrived(const pb::tagreader::Message& message) {
QStringFromStdString(req.title()),
req.size(),
QStringFromStdString(req.mime_type()),
QStringFromStdString(req.access_token()),
QStringFromStdString(req.authorisation_header()),
reply.mutable_read_google_drive_response()->mutable_metadata())) {
reply.mutable_read_google_drive_response()->clear_metadata();
}
@ -615,12 +615,12 @@ bool TagReaderWorker::ReadGoogleDrive(const QUrl& download_url,
const QString& title,
int size,
const QString& mime_type,
const QString& access_token,
const QString& authorisation_header,
pb::tagreader::SongMetadata* song) const {
qLog(Debug) << "Loading tags from" << title;
GoogleDriveStream* stream = new GoogleDriveStream(
download_url, title, size, access_token, network_);
download_url, title, size, authorisation_header, network_);
stream->Precache();
scoped_ptr<TagLib::File> tag;
if (mime_type == "audio/mpeg" && title.endsWith(".mp3")) {

View File

@ -1,16 +1,16 @@
/* This file is part of Clementine.
Copyright 2011, David Sansome <me@davidsansome.com>
Clementine is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Clementine is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Clementine. If not, see <http://www.gnu.org/licenses/>.
*/

View File

@ -86,7 +86,7 @@ message ReadGoogleDriveRequest {
optional string download_url = 1;
optional string title = 2;
optional int32 size = 3;
optional string access_token = 4;
optional string authorisation_header = 4;
optional string mime_type = 5;
}

View File

@ -37,7 +37,7 @@
#include <QVariant>
const char* Database::kDatabaseFilename = "clementine.db";
const int Database::kSchemaVersion = 39;
const int Database::kSchemaVersion = 40;
const char* Database::kMagicAllSongsTables = "%allsongstables";
int Database::sNextConnectionId = 1;

View File

@ -88,7 +88,7 @@ TagReaderReply* TagReaderClient::ReadGoogleDrive(const QUrl& download_url,
const QString& title,
int size,
const QString& mime_type,
const QString& access_token) {
const QString& authorisation_header) {
pb::tagreader::Message message;
pb::tagreader::ReadGoogleDriveRequest* req =
message.mutable_read_google_drive_request();
@ -98,7 +98,7 @@ TagReaderReply* TagReaderClient::ReadGoogleDrive(const QUrl& download_url,
req->set_title(DataCommaSizeFromQString(title));
req->set_size(size);
req->set_mime_type(DataCommaSizeFromQString(mime_type));
req->set_access_token(DataCommaSizeFromQString(access_token));
req->set_authorisation_header(DataCommaSizeFromQString(authorisation_header));
return worker_pool_->SendMessageWithReply(&message);
}

View File

@ -49,7 +49,7 @@ public:
const QString& title,
int size,
const QString& mime_type,
const QString& access_token);
const QString& authorisation_header);
// Convenience functions that call the above functions and wait for a
// response. These block the calling thread with a semaphore, and must NOT

View File

@ -178,12 +178,13 @@ void GoogleDriveService::MaybeAddFileToDatabase(const google_drive::File& file)
tr("Indexing %1").arg(file.title()));
// Song not in index; tag and add.
QString authorisation_header = QString("Bearer %1").arg(client_->access_token());
TagReaderClient::ReplyType* reply = app_->tag_reader_client()->ReadGoogleDrive(
file.download_url(),
file.title(),
file.size(),
file.mime_type(),
client_->access_token());
authorisation_header);
NewClosure(reply, SIGNAL(Finished(bool)),
this, SLOT(ReadTagsFinished(TagReaderClient::ReplyType*,google_drive::File,QString,int)),

View File

@ -2,19 +2,25 @@
#include <QDateTime>
#include <QSettings>
#include <QSortFilterProxyModel>
#include <qjson/parser.h>
#include "core/application.h"
#include "core/closure.h"
#include "core/database.h"
#include "core/logging.h"
#include "core/mergedproxymodel.h"
#include "core/network.h"
#include "core/player.h"
#include "core/timeconstants.h"
#include "core/utilities.h"
#include "globalsearch/globalsearch.h"
#include "globalsearch/librarysearchprovider.h"
#include "internet/internetmodel.h"
#include "internet/ubuntuoneauthenticator.h"
#include "internet/ubuntuoneurlhandler.h"
#include "library/librarybackend.h"
const char* UbuntuOneService::kServiceName = "Ubuntu One";
const char* UbuntuOneService::kSettingsGroup = "Ubuntu One";
@ -23,13 +29,33 @@ namespace {
static const char* kFileStorageEndpoint =
"https://one.ubuntu.com/api/file_storage/v1/~/Ubuntu One/";
static const char* kContentRoot = "https://files.one.ubuntu.com";
static const char* kSongsTable = "ubuntu_one_songs";
static const char* kFtsTable = "ubuntu_one_songs_fts";
}
UbuntuOneService::UbuntuOneService(Application* app, InternetModel* parent)
: InternetService(kServiceName, app, parent, parent),
root_(nullptr),
network_(new NetworkAccessManager(this)) {
network_(new NetworkAccessManager(this)),
library_sort_model_(new QSortFilterProxyModel(this)) {
library_backend_ = new LibraryBackend;
library_backend_->moveToThread(app_->database()->thread());
library_backend_->Init(
app->database(), kSongsTable, QString::null, QString::null, kFtsTable);
library_model_ = new LibraryModel(library_backend_, app_, this);
library_sort_model_->setSourceModel(library_model_);
library_sort_model_->setSortRole(LibraryModel::Role_SortText);
library_sort_model_->setDynamicSortFilter(true);
library_sort_model_->sort(0);
app->player()->RegisterUrlHandler(new UbuntuOneUrlHandler(this, this));
app->global_search()->AddProvider(new LibrarySearchProvider(
library_backend_,
kServiceName,
"ubuntu_one",
QIcon(":/providers/ubuntuone.png"),
true, app_, this));
}
QStandardItem* UbuntuOneService::CreateRootItem() {
@ -42,6 +68,8 @@ void UbuntuOneService::LazyPopulate(QStandardItem* item) {
switch (item->data(InternetModel::Role_Type).toInt()) {
case InternetModel::Type_Service:
Connect();
library_model_->Init();
model()->merged_model()->AddSubModel(item->index(), library_sort_model_);
break;
default:
@ -125,20 +153,58 @@ void UbuntuOneService::FileListRequestFinished(QNetworkReply* reply) {
QVariantList children = result["children"].toList();
for (const QVariant& c : children) {
QVariantMap child = c.toMap();
QString content_path = child["content_path"].toString();
QUrl content_url;
content_url.setScheme("ubuntuonefile");
content_url.setPath(content_path);
Song song;
song.set_title(child["path"].toString().mid(1));
song.set_url(content_url);
root_->appendRow(CreateSongItem(song));
MaybeAddFileToDatabase(child);
}
}
void UbuntuOneService::MaybeAddFileToDatabase(const QVariantMap& file) {
QString content_path = file["content_path"].toString();
QUrl service_url;
service_url.setScheme("ubuntuonefile");
service_url.setPath(content_path);
Song song = library_backend_->GetSongByUrl(service_url);
if (song.is_valid()) {
return;
}
QUrl content_url(kContentRoot);
content_url.setPath(content_path);
TagReaderClient::ReplyType* reply = app_->tag_reader_client()->ReadGoogleDrive(
content_url,
file["path"].toString().mid(1),
file["size"].toInt(),
"audio/mpeg",
GenerateAuthorisationHeader());
NewClosure(
reply, SIGNAL(Finished(bool)),
this, SLOT(ReadTagsFinished(TagReaderClient::ReplyType*,QVariantMap, QUrl)),
reply, file, service_url);
}
void UbuntuOneService::ReadTagsFinished(
TagReaderClient::ReplyType* reply, const QVariantMap& file, const QUrl& url) {
qLog(Debug) << reply->message().DebugString().c_str();
Song song;
song.InitFromProtobuf(reply->message().read_google_drive_response().metadata());
song.set_directory_id(0);
song.set_etag(file["hash"].toString());
song.set_mtime(
QDateTime::fromString(file["when_changed"].toString(), Qt::ISODate).toTime_t());
song.set_ctime(
QDateTime::fromString(file["when_created"].toString(), Qt::ISODate).toTime_t());
song.set_url(url);
if (song.title().isEmpty()) {
song.set_title(file["path"].toString().mid(1));
}
qLog(Debug) << "Adding song to db:" << song.title();
library_backend_->AddOrUpdateSongs(SongList() << song);
}
QUrl UbuntuOneService::GetStreamingUrlFromSongId(const QString& song_id) {
QUrl url(kContentRoot);
url.setPath(song_id);

View File

@ -3,8 +3,13 @@
#include "internet/internetservice.h"
#include "core/tagreaderclient.h"
class LibraryBackend;
class LibraryModel;
class NetworkAccessManager;
class QNetworkReply;
class QSortFilterProxyModel;
class UbuntuOneAuthenticator;
class UbuntuOneService : public InternetService {
@ -24,10 +29,13 @@ class UbuntuOneService : public InternetService {
private slots:
void AuthenticationFinished(UbuntuOneAuthenticator* authenticator);
void FileListRequestFinished(QNetworkReply* reply);
void ReadTagsFinished(
TagReaderClient::ReplyType* reply, const QVariantMap& file, const QUrl& url);
private:
void Connect();
void RequestFileList();
void MaybeAddFileToDatabase(const QVariantMap& file);
private:
QByteArray GenerateAuthorisationHeader();
@ -39,6 +47,10 @@ class UbuntuOneService : public InternetService {
QString consumer_secret_;
QString token_;
QString token_secret_;
LibraryBackend* library_backend_;
LibraryModel* library_model_;
QSortFilterProxyModel* library_sort_model_;
};
#endif // UBUNTUONESERVICE_H