Read Google Drive metadata in the tagreader worker process

This commit is contained in:
David Sansome 2012-07-28 19:35:12 +01:00
parent 165cec1e86
commit 9653a45f66
12 changed files with 370 additions and 186 deletions

View File

@ -4,6 +4,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
include_directories(${CMAKE_SOURCE_DIR}/ext/libclementine-common)
include_directories(${CMAKE_BINARY_DIR}/ext/libclementine-tagreader)
include_directories(${CMAKE_SOURCE_DIR}/src)
include_directories(${CMAKE_BINARY_DIR}/src)
set(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR})
@ -16,6 +17,12 @@ set(SOURCES
set(HEADERS
)
optional_source(HAVE_GOOGLE_DRIVE
INCLUDE_DIRECTORIES ${SPARSEHASH_INCLUDE_DIRS}
SOURCES
googledrivestream.cpp
)
qt4_wrap_cpp(MOC ${HEADERS})
add_executable(clementine-tagreader

View File

@ -0,0 +1,157 @@
/* This file is part of Clementine.
Copyright 2012, David Sansome <me@davidsansome.com>
Clementine is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Clementine is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Clementine. If not, see <http://www.gnu.org/licenses/>.
*/
#include "googledrivestream.h"
#include "core/logging.h"
#include <QEventLoop>
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QNetworkRequest>
#include <taglib/id3v2framefactory.h>
#include <taglib/mpegfile.h>
using TagLib::ByteVector;
GoogleDriveStream::GoogleDriveStream(
const QUrl& url, const QString& filename, const long length,
const QString& auth, QNetworkAccessManager* network)
: url_(url),
filename_(filename),
encoded_filename_(filename_.toUtf8()),
length_(length),
auth_(auth),
cursor_(0),
network_(network),
cache_(length) {
}
TagLib::FileName GoogleDriveStream::name() const {
return encoded_filename_.data();
}
bool GoogleDriveStream::CheckCache(int start, int end) {
for (int i = start; i <= end; ++i) {
if (!cache_.test(i)) {
return false;
}
}
return true;
}
void GoogleDriveStream::FillCache(int start, TagLib::ByteVector data) {
for (int i = 0; i < data.size(); ++i) {
cache_.set(start + i, data[i]);
}
}
TagLib::ByteVector GoogleDriveStream::GetCached(int start, int end) {
const uint size = end - start + 1;
TagLib::ByteVector ret(size);
for (int i = 0; i < size; ++i) {
ret[i] = cache_.get(start + i);
}
return ret;
}
TagLib::ByteVector GoogleDriveStream::readBlock(ulong length) {
const uint start = cursor_;
const uint end = qMin(cursor_ + length - 1, length_ - 1);
if (end <= start) {
return TagLib::ByteVector();
}
if (CheckCache(start, end)) {
TagLib::ByteVector cached = GetCached(start, end);
cursor_ += cached.size();
return cached;
}
QNetworkRequest request = QNetworkRequest(url_);
request.setRawHeader(
"Authorization", QString("Bearer %1").arg(auth_).toUtf8());
request.setRawHeader(
"Range", QString("bytes=%1-%2").arg(start).arg(end).toUtf8());
QNetworkReply* reply = network_->get(request);
QEventLoop loop;
QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit()));
loop.exec();
reply->deleteLater();
QByteArray data = reply->readAll();
TagLib::ByteVector bytes(data.data(), data.size());
cursor_ += data.size();
FillCache(start, bytes);
return bytes;
}
void GoogleDriveStream::writeBlock(const ByteVector&) {
qLog(Debug) << Q_FUNC_INFO << "not implemented";
}
void GoogleDriveStream::insert(const ByteVector&, ulong, ulong) {
qLog(Debug) << Q_FUNC_INFO << "not implemented";
}
void GoogleDriveStream::removeBlock(ulong, ulong) {
qLog(Debug) << Q_FUNC_INFO << "not implemented";
}
bool GoogleDriveStream::readOnly() const {
qLog(Debug) << Q_FUNC_INFO;
return true;
}
bool GoogleDriveStream::isOpen() const {
return true;
}
void GoogleDriveStream::seek(long offset, TagLib::IOStream::Position p) {
switch (p) {
case TagLib::IOStream::Beginning:
cursor_ = offset;
break;
case TagLib::IOStream::Current:
cursor_ = qMin(ulong(cursor_ + offset), length_);
break;
case TagLib::IOStream::End:
cursor_ = qMax(0UL, length_ - offset);
break;
}
}
void GoogleDriveStream::clear() {
cursor_ = 0;
}
long GoogleDriveStream::tell() const {
return cursor_;
}
long GoogleDriveStream::length() {
return length_;
}
void GoogleDriveStream::truncate(long) {
qLog(Debug) << Q_FUNC_INFO << "not implemented";
}

View File

@ -0,0 +1,68 @@
/* This file is part of Clementine.
Copyright 2012, David Sansome <me@davidsansome.com>
Clementine is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Clementine is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Clementine. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef GOOGLEDRIVESTREAM_H
#define GOOGLEDRIVESTREAM_H
#include <QUrl>
#include <google/sparsetable>
#include <taglib/tiostream.h>
class QNetworkAccessManager;
class GoogleDriveStream : public TagLib::IOStream {
public:
GoogleDriveStream(const QUrl& url,
const QString& filename,
const long length,
const QString& auth,
QNetworkAccessManager* network);
// Taglib::IOStream
virtual TagLib::FileName name() const;
virtual TagLib::ByteVector readBlock(ulong length);
virtual void writeBlock(const TagLib::ByteVector&);
virtual void insert(const TagLib::ByteVector&, ulong, ulong);
virtual void removeBlock(ulong, ulong);
virtual bool readOnly() const;
virtual bool isOpen() const;
virtual void seek(long offset, TagLib::IOStream::Position p);
virtual void clear();
virtual long tell() const;
virtual long length();
virtual void truncate(long);
private:
bool CheckCache(int start, int end);
void FillCache(int start, TagLib::ByteVector data);
TagLib::ByteVector GetCached(int start, int end);
private:
const QUrl url_;
const QString filename_;
const QByteArray encoded_filename_;
const ulong length_;
const QString auth_;
int cursor_;
QNetworkAccessManager* network_;
google::sparsetable<char> cache_;
};
#endif // GOOGLEDRIVESTREAM_H

View File

@ -24,6 +24,7 @@
#include <QCoreApplication>
#include <QDateTime>
#include <QFileInfo>
#include <QNetworkAccessManager>
#include <QTextCodec>
#include <QUrl>
@ -56,6 +57,10 @@
# define TAGLIB_HAS_FLAC_PICTURELIST
#endif
#ifdef HAVE_GOOGLE_DRIVE
# include "googledrivestream.h"
#endif
using boost::scoped_ptr;
@ -93,6 +98,7 @@ TagLib::String QStringToTaglibString(const QString& s) {
TagReaderWorker::TagReaderWorker(QIODevice* socket, QObject* parent)
: AbstractMessageHandler<pb::tagreader::Message>(socket, parent),
factory_(new TagLibFileRefFactory),
network_(new QNetworkAccessManager),
kEmbeddedCover("(embedded)")
{
}
@ -123,6 +129,17 @@ void TagReaderWorker::MessageArrived(const pb::tagreader::Message& message) {
QStringFromStdString(message.load_embedded_art_request().filename()));
reply.mutable_load_embedded_art_response()->set_data(
data.constData(), data.size());
} else if (message.has_read_google_drive_request()) {
#ifdef HAVE_GOOGLE_DRIVE
const pb::tagreader::ReadGoogleDriveRequest& req =
message.read_google_drive_request();
ReadGoogleDrive(QUrl::fromEncoded(QByteArray(req.download_url().data(),
req.download_url().size())),
QStringFromStdString(req.title()),
req.size(),
QStringFromStdString(req.access_token()),
reply.mutable_read_google_drive_response()->mutable_metadata());
#endif
}
SendReply(message, &reply);
@ -588,3 +605,32 @@ void TagReaderWorker::DeviceClosed() {
qApp->exit();
}
#ifdef HAVE_GOOGLE_DRIVE
void TagReaderWorker::ReadGoogleDrive(const QUrl& download_url,
const QString& title,
int size,
const QString& access_token,
pb::tagreader::SongMetadata* song) const {
qLog(Debug) << "Loading tags from" << title;
GoogleDriveStream* stream = new GoogleDriveStream(
download_url, title, size, access_token, network_);
TagLib::MPEG::File tag(
stream, // Takes ownership.
TagLib::ID3v2::FrameFactory::instance(),
TagLib::AudioProperties::Fast);
if (tag.tag()) {
song->set_title(tag.tag()->title().toCString(true));
song->set_artist(tag.tag()->artist().toCString(true));
song->set_album(tag.tag()->album().toCString(true));
song->set_filesize(size);
song->set_type(pb::tagreader::SongMetadata_Type_STREAM);
if (tag.audioProperties()) {
song->set_length_nanosec(tag.audioProperties()->length() * kNsecPerSec);
}
}
}
#endif // HAVE_GOOGLE_DRIVE

View File

@ -18,11 +18,16 @@
#ifndef TAGREADERWORKER_H
#define TAGREADERWORKER_H
#include "config.h"
#include "tagreadermessages.pb.h"
#include "core/messagehandler.h"
#include <taglib/xiphcomment.h>
#include <QUrl>
class QNetworkAccessManager;
namespace TagLib {
class FileRef;
@ -49,6 +54,14 @@ private:
bool IsMediaFile(const QString& filename) const;
QByteArray LoadEmbeddedArt(const QString& filename) const;
#ifdef HAVE_GOOGLE_DRIVE
void ReadGoogleDrive(const QUrl& download_url,
const QString& title,
int size,
const QString& access_token,
pb::tagreader::SongMetadata* song) const;
#endif // HAVE_GOOGLE_DRIVE
static void Decode(const TagLib::String& tag, const QTextCodec* codec,
std::string* output);
static void Decode(const QString& tag, const QTextCodec* codec,
@ -69,6 +82,7 @@ private:
private:
FileRefFactory* factory_;
QNetworkAccessManager* network_;
const std::string kEmbeddedCover;
};

View File

@ -85,6 +85,17 @@ message LoadEmbeddedArtResponse {
optional bytes data = 1;
}
message ReadGoogleDriveRequest {
optional string download_url = 1;
optional string title = 2;
optional int32 size = 3;
optional string access_token = 4;
}
message ReadGoogleDriveResponse {
optional SongMetadata metadata = 1;
}
message Message {
optional int32 id = 1;
@ -99,4 +110,7 @@ message Message {
optional LoadEmbeddedArtRequest load_embedded_art_request = 8;
optional LoadEmbeddedArtResponse load_embedded_art_response = 9;
optional ReadGoogleDriveRequest read_google_drive_request = 10;
optional ReadGoogleDriveResponse read_google_drive_response = 11;
}

View File

@ -966,7 +966,6 @@ optional_source(HAVE_LIBMTP
# Windows media lister
optional_source(HAVE_SAC
INCLUDE_DIRECTORIES ${SPARSEHASH_INCLUDE_DIRS}
SOURCES
devices/wmdmdevice.cpp
devices/wmdmlister.cpp

View File

@ -21,6 +21,7 @@
#include <QFile>
#include <QProcess>
#include <QTcpServer>
#include <QUrl>
const char* TagReaderClient::kWorkerExecutableName = "clementine-tagreader";
@ -83,6 +84,23 @@ TagReaderReply* TagReaderClient::LoadEmbeddedArt(const QString& filename) {
return worker_pool_->SendMessageWithReply(&message);
}
TagReaderReply* TagReaderClient::ReadGoogleDrive(const QUrl& download_url,
const QString& title,
int size,
const QString& access_token) {
pb::tagreader::Message message;
pb::tagreader::ReadGoogleDriveRequest* req =
message.mutable_read_google_drive_request();
const QString url_string = download_url.toEncoded();
req->set_download_url(DataCommaSizeFromQString(url_string));
req->set_title(DataCommaSizeFromQString(title));
req->set_size(size);
req->set_access_token(DataCommaSizeFromQString(access_token));
return worker_pool_->SendMessageWithReply(&message);
}
void TagReaderClient::ReadFileBlocking(const QString& filename, Song* song) {
Q_ASSERT(QThread::currentThread() != thread());

View File

@ -45,6 +45,10 @@ public:
ReplyType* SaveFile(const QString& filename, const Song& metadata);
ReplyType* IsMediaFile(const QString& filename);
ReplyType* LoadEmbeddedArt(const QString& filename);
ReplyType* ReadGoogleDrive(const QUrl& download_url,
const QString& title,
int size,
const QString& access_token);
// Convenience functions that call the above functions and wait for a
// response. These block the calling thread with a semaphore, and must NOT

View File

@ -42,6 +42,7 @@ public:
QString id() const { return data_["id"].toString(); }
QString etag() const { return data_["etag"].toString(); }
QString title() const { return data_["title"].toString(); }
QString description() const { return data_["description"].toString(); }
long size() const { return data_["fileSize"].toUInt(); }
QUrl download_url() const { return data_["downloadUrl"].toUrl(); }

View File

@ -4,13 +4,6 @@
#include <QScopedPointer>
#include <QSortFilterProxyModel>
#include <google/sparsetable>
#include <taglib/id3v2framefactory.h>
#include <taglib/mpegfile.h>
#include <taglib/tiostream.h>
using TagLib::ByteVector;
#include "core/application.h"
#include "core/closure.h"
#include "core/database.h"
@ -35,153 +28,6 @@ static const char* kFtsTable = "google_drive_songs_fts";
}
class DriveStream : public TagLib::IOStream {
public:
DriveStream(const QUrl& url,
const QString& filename,
const long length,
const QString& auth,
QNetworkAccessManager* network)
: url_(url),
filename_(filename),
encoded_filename_(filename_.toUtf8()),
length_(length),
auth_(auth),
cursor_(0),
network_(network),
cache_(length) {
}
virtual TagLib::FileName name() const {
return encoded_filename_.data();
}
bool CheckCache(int start, int end) {
for (int i = start; i <= end; ++i) {
if (!cache_.test(i)) {
return false;
}
}
return true;
}
void FillCache(int start, TagLib::ByteVector data) {
for (int i = 0; i < data.size(); ++i) {
cache_.set(start + i, data[i]);
}
}
TagLib::ByteVector GetCached(int start, int end) {
const uint size = end - start + 1;
TagLib::ByteVector ret(size);
for (int i = 0; i < size; ++i) {
ret[i] = cache_.get(start + i);
}
return ret;
}
virtual TagLib::ByteVector readBlock(ulong length) {
const uint start = cursor_;
const uint end = qMin(cursor_ + length - 1, length_ - 1);
if (end <= start) {
return TagLib::ByteVector();
}
if (CheckCache(start, end)) {
TagLib::ByteVector cached = GetCached(start, end);
cursor_ += cached.size();
return cached;
}
QNetworkRequest request = QNetworkRequest(url_);
request.setRawHeader(
"Authorization", QString("Bearer %1").arg(auth_).toUtf8());
request.setRawHeader(
"Range", QString("bytes=%1-%2").arg(start).arg(end).toUtf8());
QNetworkReply* reply = network_->get(request);
QEventLoop loop;
QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit()));
loop.exec();
reply->deleteLater();
QByteArray data = reply->readAll();
TagLib::ByteVector bytes(data.data(), data.size());
cursor_ += data.size();
FillCache(start, bytes);
return bytes;
}
virtual void writeBlock(const ByteVector&) {
qLog(Debug) << Q_FUNC_INFO << "not implemented";
}
virtual void insert(const ByteVector&, ulong, ulong) {
qLog(Debug) << Q_FUNC_INFO << "not implemented";
}
virtual void removeBlock(ulong, ulong) {
qLog(Debug) << Q_FUNC_INFO << "not implemented";
}
virtual bool readOnly() const {
qLog(Debug) << Q_FUNC_INFO;
return true;
}
virtual bool isOpen() const {
return true;
}
virtual void seek(long offset, TagLib::IOStream::Position p) {
switch (p) {
case TagLib::IOStream::Beginning:
cursor_ = offset;
break;
case TagLib::IOStream::Current:
cursor_ = qMin(ulong(cursor_ + offset), length_);
break;
case TagLib::IOStream::End:
cursor_ = qMax(0UL, length_ - offset);
break;
}
}
virtual void clear() {
cursor_ = 0;
}
virtual long tell() const {
return cursor_;
}
virtual long length() {
return length_;
}
virtual void truncate(long) {
qLog(Debug) << Q_FUNC_INFO << "not implemented";
}
private:
const QUrl url_;
const QString filename_;
const QByteArray encoded_filename_;
const ulong length_;
const QString auth_;
int cursor_;
QNetworkAccessManager* network_;
google::sparsetable<char> cache_;
};
GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent)
: InternetService("Google Drive", app, parent, parent),
root_(NULL),
@ -199,7 +45,7 @@ GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent)
library_sort_model_->sort(0);
app->player()->RegisterUrlHandler(new GoogleDriveUrlHandler(this, this));
app_->global_search()->AddProvider(new LibrarySearchProvider(
app->global_search()->AddProvider(new LibrarySearchProvider(
library_backend_,
tr("Google Drive"),
"google_drive",
@ -275,43 +121,49 @@ void GoogleDriveService::MaybeAddFileToDatabase(const google_drive::File& file)
}
// Song not in index; tag and add.
DriveStream* stream = new DriveStream(
TagReaderClient::ReplyType* reply = app_->tag_reader_client()->ReadGoogleDrive(
file.download_url(),
file.title(),
file.size(),
client_->access_token(),
&network_);
TagLib::MPEG::File tag(
stream, // Takes ownership.
TagLib::ID3v2::FrameFactory::instance(),
TagLib::AudioProperties::Fast);
if (tag.tag()) {
Song song;
song.set_title(tag.tag()->title().toCString(true));
song.set_artist(tag.tag()->artist().toCString(true));
song.set_album(tag.tag()->album().toCString(true));
client_->access_token());
song.set_url(url);
song.set_filesize(file.size());
song.set_etag(file.etag().remove('"'));
NewClosure(reply, SIGNAL(Finished(bool)),
this, SLOT(ReadTagsFinished(TagReaderClient::ReplyType*,google_drive::File,QString)),
reply, file, url);
}
song.set_mtime(file.modified_date().toTime_t());
song.set_ctime(file.created_date().toTime_t());
void GoogleDriveService::ReadTagsFinished(TagReaderClient::ReplyType* reply,
const google_drive::File& metadata,
const QString& url) {
reply->deleteLater();
song.set_filetype(Song::Type_Stream);
song.set_directory_id(0);
if (tag.audioProperties()) {
song.set_length_nanosec(tag.audioProperties()->length() * kNsecPerSec);
}
SongList songs;
songs << song;
qLog(Debug) << "Adding song to db:" << song.title();
library_backend_->AddOrUpdateSongs(songs);
} else {
qLog(Debug) << "Failed to tag:" << url;
const pb::tagreader::ReadGoogleDriveResponse& msg =
reply->message().read_google_drive_response();
if (!msg.metadata().filesize()) {
qLog(Debug) << "Failed to tag:" << metadata.download_url();
return;
}
// Read the Song metadata from the message.
Song song;
song.InitFromProtobuf(msg.metadata());
// Add some extra tags from the Google Drive metadata.
song.set_etag(metadata.etag().remove('"'));
song.set_mtime(metadata.modified_date().toTime_t());
song.set_ctime(metadata.created_date().toTime_t());
song.set_comment(metadata.description());
song.set_directory_id(0);
song.set_url(url);
// Use the Google Drive title if we couldn't read tags from the file.
if (song.title().isEmpty()) {
song.set_title(metadata.title());
}
// Add the song to the database
qLog(Debug) << "Adding song to db:" << song.title();
library_backend_->AddOrUpdateSongs(SongList() << song);
}
QUrl GoogleDriveService::GetStreamingUrlFromSongId(const QString& id) {

View File

@ -4,6 +4,7 @@
#include "internetservice.h"
#include "core/network.h"
#include "core/tagreaderclient.h"
class QStandardItem;
@ -32,6 +33,9 @@ class GoogleDriveService : public InternetService {
void ConnectFinished(google_drive::ConnectResponse* response);
void FilesFound(const QList<google_drive::File>& files);
void ListFilesFinished(google_drive::ListFilesResponse* response);
void ReadTagsFinished(TagReaderClient::ReplyType* reply,
const google_drive::File& metadata,
const QString& url);
private:
void Connect();