finished google drive sync w/o conflict resolver

This commit is contained in:
Mariotaku Lee 2017-01-22 14:53:06 +08:00
parent 077184deae
commit 7eb3fb7cf1
No known key found for this signature in database
GPG Key ID: 9C0706AE47FCE2AD
11 changed files with 218 additions and 83 deletions

View File

@ -24,6 +24,7 @@ import android.os.Parcelable;
import android.support.annotation.Nullable;
import android.support.annotation.StringDef;
import com.hannesdorfmann.parcelableplease.annotation.ParcelableNoThanks;
import com.hannesdorfmann.parcelableplease.annotation.ParcelablePlease;
import com.hannesdorfmann.parcelableplease.annotation.ParcelableThisPlease;
@ -77,6 +78,13 @@ public class Draft implements Parcelable {
@CursorField(value = Drafts.UNIQUE_ID)
public String unique_id;
/**
* For internal use only
*/
@Nullable
@ParcelableNoThanks
public String remote_extras;
public Draft() {
}

View File

@ -30,7 +30,7 @@ class GoogleDriveAuthActivity : BaseActivity(), GoogleApiClient.ConnectionCallba
super.onCreate(savedInstanceState)
val gso = GoogleSignInOptions.Builder(GoogleSignInOptions.DEFAULT_SIGN_IN)
.requestScopes(Scope(DriveScopes.DRIVE), Scope(DriveScopes.DRIVE_METADATA))
.requestScopes(Scope(DriveScopes.DRIVE))
.requestServerAuthCode(GoogleDriveSyncProviderInfo.WEB_CLIENT_ID, true)
.build()

View File

@ -3,6 +3,7 @@ package org.mariotaku.twidere.util.sync.google
import android.content.Context
import com.google.api.client.util.DateTime
import com.google.api.services.drive.Drive
import com.google.api.services.drive.model.File
import org.mariotaku.twidere.extension.model.filename
import org.mariotaku.twidere.extension.model.readMimeMessageFrom
import org.mariotaku.twidere.extension.model.writeMimeMessageTo
@ -28,20 +29,28 @@ internal class GoogleDriveDraftsSyncAction(
override fun Draft.saveToRemote(): DriveFileInfo {
val os = DirectByteArrayOutputStream()
this.writeMimeMessageTo(context, os)
val file = files.updateOrCreate(filename, draftMimeType, folderId, stream = os.inputStream(true), fileConfig = {
val driveId = this.remote_extras
val `is` = os.inputStream(true)
val fileConfig: (File) -> Unit = {
it.modifiedTime = DateTime(timestamp)
})
return DriveFileInfo(file.id, file.name, Date(timestamp))
}
val file = if (driveId != null) {
drive.files().performUpdate(driveId, filename, draftMimeType, stream = `is`, fileConfig = fileConfig)
} else {
drive.updateOrCreate(filename, draftMimeType, folderId, stream = `is`, fileConfig = fileConfig)
}
return DriveFileInfo(file.id, file.name, Date(file.modifiedTime.value))
}
@Throws(IOException::class)
override fun Draft.loadFromRemote(info: DriveFileInfo): Boolean {
val get = files.get(info.fileId)
get.executeAsInputStream().use {
get.executeMediaAsInputStream().use {
val parsed = this.readMimeMessageFrom(context, it)
if (parsed) {
this.timestamp = info.draftTimestamp
this.unique_id = info.draftFileName.substringBeforeLast(".eml")
this.remote_extras = info.fileId
}
return parsed
}
@ -49,9 +58,12 @@ internal class GoogleDriveDraftsSyncAction(
@Throws(IOException::class)
override fun removeDrafts(list: List<DriveFileInfo>): Boolean {
val batch = drive.batch()
val callback = SimpleJsonBatchCallback<Void>()
list.forEach { info ->
files.delete(info.fileId).execute()
files.delete(info.fileId).queue(batch, callback)
}
batch.execute()
return true
}
@ -65,27 +77,29 @@ internal class GoogleDriveDraftsSyncAction(
override val DriveFileInfo.draftFileName: String get() = this.name
override val DriveFileInfo.draftRemoteExtras: String? get() = this.fileId
@Throws(IOException::class)
override fun listRemoteDrafts(): List<DriveFileInfo> {
val result = ArrayList<DriveFileInfo>()
var pageToken: String?
do {
val executeResult = files.list().apply {
val listResult = files.list().apply {
fields = "files($requiredRequestFields)"
q = "'$folderId' in parents and mimeType = '$draftMimeType' and trashed = false"
}.execute()
executeResult.files.filter { file ->
listResult.files.filter { file ->
file.mimeType == draftMimeType
}.mapTo(result) { file ->
val lastModified = file.modifiedTime ?: file.createdTime
DriveFileInfo(file.id, file.name, Date(lastModified?.value ?: 0))
DriveFileInfo(file.id, file.name, Date(file.modifiedTime.value))
}
pageToken = executeResult.nextPageToken
pageToken = listResult.nextPageToken
} while (pageToken != null)
return result
}
override fun setup(): Boolean {
folderId = files.getOrCreate(draftsDirName, folderMimeType).id
folderId = drive.getFileOrCreate(draftsDirName, folderMimeType, conflictResolver = ::resolveFoldersConflict).id
return true
}

View File

@ -12,6 +12,7 @@ import org.mariotaku.twidere.model.FiltersData
import org.mariotaku.twidere.util.io.DirectByteArrayOutputStream
import org.mariotaku.twidere.util.sync.FileBasedFiltersDataSyncAction
import java.io.FileNotFoundException
import java.io.IOException
import java.io.InputStream
internal class GoogleDriveFiltersDataSyncAction(
@ -25,17 +26,20 @@ internal class GoogleDriveFiltersDataSyncAction(
private val files = drive.files()
override fun newLoadFromRemoteSession(): CloseableAny<File> {
val file = files.getOrNull(fileName, xmlMimeType, commonFolderId) ?: throw FileNotFoundException()
val file = drive.getFileOrNull(fileName, xmlMimeType, commonFolderId,
conflictResolver = ::resolveFilesConflict) ?: run {
throw FileNotFoundException()
}
return CloseableAny(file)
}
override fun CloseableAny<File>.getRemoteLastModified(): Long {
return (obj.modifiedTime ?: obj.createdTime)?.value ?: 0
return obj.modifiedTime?.value ?: throw IOException("Modified time should not be null")
}
override fun CloseableAny<File>.loadFromRemote(): FiltersData {
val data = FiltersData()
data.parse(files.get(obj.id).executeAsInputStream().newPullParser(charset = Charsets.UTF_8))
data.parse(files.get(obj.id).executeMediaAsInputStream().newPullParser(charset = Charsets.UTF_8))
data.initFields()
return data
}
@ -49,7 +53,7 @@ internal class GoogleDriveFiltersDataSyncAction(
}
override fun newSaveToRemoteSession(): GoogleDriveUploadSession<FiltersData> {
return object : GoogleDriveUploadSession<FiltersData>(fileName, commonFolderId, xmlMimeType, files) {
return object : GoogleDriveUploadSession<FiltersData>(fileName, commonFolderId, xmlMimeType, drive) {
override fun FiltersData.toInputStream(): InputStream {
val os = DirectByteArrayOutputStream()
this.serialize(os.newSerializer(charset = Charsets.UTF_8, indent = true))
@ -60,7 +64,7 @@ internal class GoogleDriveFiltersDataSyncAction(
override fun setup(): Boolean {
commonFolderId = files.getOrCreate("Common", folderMimeType).id
commonFolderId = drive.getFileOrCreate("Common", folderMimeType, conflictResolver = ::resolveFoldersConflict).id
return true
}

View File

@ -2,19 +2,14 @@ package org.mariotaku.twidere.util.sync.google
import android.content.Context
import android.content.SharedPreferences
import com.dropbox.core.DbxDownloader
import com.dropbox.core.v2.DbxClientV2
import com.dropbox.core.v2.files.FileMetadata
import com.dropbox.core.v2.files.UploadUploader
import com.google.api.services.drive.Drive
import com.google.api.services.drive.model.File
import org.mariotaku.twidere.extension.model.serialize
import org.mariotaku.twidere.extension.newPullParser
import org.mariotaku.twidere.extension.newSerializer
import org.mariotaku.twidere.model.FiltersData
import org.mariotaku.twidere.util.io.DirectByteArrayOutputStream
import org.mariotaku.twidere.util.sync.FileBasedPreferencesValuesSyncAction
import java.io.FileNotFoundException
import java.io.IOException
import java.io.InputStream
import java.util.*
@ -32,22 +27,25 @@ internal class GoogleDrivePreferencesValuesSyncAction(
private val files = drive.files()
override fun newLoadFromRemoteSession(): CloseableAny<File> {
val file = files.getOrNull(fileName, xmlMimeType, commonFolderId) ?: throw FileNotFoundException()
val file = drive.getFileOrNull(fileName, xmlMimeType, commonFolderId,
conflictResolver = ::resolveFilesConflict) ?: run {
throw FileNotFoundException()
}
return CloseableAny(file)
}
override fun CloseableAny<File>.getRemoteLastModified(): Long {
return (obj.modifiedTime ?: obj.createdTime)?.value ?: 0
return obj.modifiedTime?.value ?: throw IOException("Modified time should not be null")
}
override fun CloseableAny<File>.loadFromRemote(): MutableMap<String, String> {
val data = HashMap<String, String>()
data.parse(files.get(obj.id).executeAsInputStream().newPullParser())
data.parse(files.get(obj.id).executeMediaAsInputStream().newPullParser())
return data
}
override fun newSaveToRemoteSession(): GoogleDriveUploadSession<Map<String, String>> {
return object : GoogleDriveUploadSession<Map<String, String>>(fileName, commonFolderId, xmlMimeType, files) {
return object : GoogleDriveUploadSession<Map<String, String>>(fileName, commonFolderId, xmlMimeType, drive) {
override fun Map<String, String>.toInputStream(): InputStream {
val os = DirectByteArrayOutputStream()
this.serialize(os.newSerializer(charset = Charsets.UTF_8, indent = true))
@ -66,7 +64,7 @@ internal class GoogleDrivePreferencesValuesSyncAction(
}
override fun setup(): Boolean {
commonFolderId = files.getOrCreate("Common", folderMimeType).id
commonFolderId = drive.getFileOrCreate("Common", folderMimeType, conflictResolver = ::resolveFoldersConflict).id
return true
}
}

View File

@ -6,6 +6,7 @@ import com.google.api.services.drive.Drive
import com.google.api.services.drive.model.File
import java.io.InputStream
/**
* Created by mariotaku on 1/21/17.
*/
@ -13,11 +14,29 @@ import java.io.InputStream
internal const val folderMimeType = "application/vnd.google-apps.folder"
internal const val xmlMimeType = "application/xml"
internal const val requiredRequestFields = "id, name, mimeType, modifiedTime"
internal fun Drive.getFileOrNull(
name: String,
mimeType: String?,
parent: String? = "root",
trashed: Boolean = false,
conflictResolver: ((Drive, List<File>) -> File)? = null
): File? {
val result = findFilesOrNull(name, mimeType, parent, trashed) ?: return null
if (result.size > 1 && conflictResolver != null) {
return conflictResolver(this, result)
}
return result.firstOrNull()
}
internal fun Drive.Files.getOrNull(name: String, mimeType: String?, parent: String? = "root",
trashed: Boolean = false): File? {
val find = list()
internal fun Drive.findFilesOrNull(
name: String,
mimeType: String?,
parent: String? = "root",
trashed: Boolean = false
): List<File>? {
val find = files().list()
var query = "name = '$name'"
if (parent != null) {
query += " and '$parent' in parents"
@ -27,8 +46,9 @@ internal fun Drive.Files.getOrNull(name: String, mimeType: String?, parent: Stri
}
query += " and trashed = $trashed"
find.q = query
find.fields = "files($requiredRequestFields)"
try {
return find.execute().files.firstOrNull()
return find.execute().files
} catch (e: GoogleJsonResponseException) {
if (e.statusCode == 404) {
return null
@ -38,18 +58,37 @@ internal fun Drive.Files.getOrNull(name: String, mimeType: String?, parent: Stri
}
}
internal fun Drive.Files.getOrCreate(name: String, mimeType: String, parent: String = "root",
trashed: Boolean = false): File {
return getOrNull(name, mimeType, parent, trashed) ?: run {
val fileMetadata = File()
fileMetadata.name = name
fileMetadata.mimeType = mimeType
fileMetadata.parents = listOf(parent)
return@run create(fileMetadata).execute()
internal fun Drive.getFileOrCreate(
name: String,
mimeType: String,
parent: String = "root",
trashed: Boolean = false,
conflictResolver: ((Drive, List<File>) -> File)? = null
): File {
val result = findFilesOrCreate(name, mimeType, parent, trashed)
if (result.size > 1 && conflictResolver != null) {
return conflictResolver(this, result)
}
return result.first()
}
internal fun Drive.findFilesOrCreate(
name: String,
mimeType: String,
parent: String = "root",
trashed: Boolean = false
): List<File> {
return findFilesOrNull(name, mimeType, parent, trashed) ?: run {
val file = File()
file.name = name
file.mimeType = mimeType
file.parents = listOf(parent)
val create = files().create(file)
return@run listOf(create.execute())
}
}
internal fun Drive.Files.updateOrCreate(
internal fun Drive.updateOrCreate(
name: String,
mimeType: String,
parent: String = "root",
@ -57,13 +96,12 @@ internal fun Drive.Files.updateOrCreate(
stream: InputStream,
fileConfig: ((file: File) -> Unit)? = null
): File {
val files = files()
return run {
val find = list()
val find = files.list()
find.q = "name = '$name' and '$parent' in parents and mimeType = '$mimeType' and trashed = $trashed"
try {
val file = find.execute().files.firstOrNull() ?: return@run null
fileConfig?.invoke(file)
return@run update(file.id, file, InputStreamContent(mimeType, stream)).execute()
val fileId = try {
find.execute().files.firstOrNull()?.id ?: return@run null
} catch (e: GoogleJsonResponseException) {
if (e.statusCode == 404) {
return@run null
@ -71,12 +109,47 @@ internal fun Drive.Files.updateOrCreate(
throw e
}
}
return@run files.performUpdate(fileId, name, mimeType, stream, fileConfig)
} ?: run {
val file = File()
file.name = name
file.mimeType = mimeType
file.parents = listOf(parent)
fileConfig?.invoke(file)
return@run create(file, InputStreamContent(mimeType, stream)).execute()
val create = files.create(file, InputStreamContent(mimeType, stream))
return@run create.execute()
}
}
internal fun Drive.Files.performUpdate(
fileId: String,
name: String,
mimeType: String,
stream: InputStream,
fileConfig: ((file: File) -> Unit)? = null
): File {
val file = File()
file.name = name
file.mimeType = mimeType
fileConfig?.invoke(file)
val update = update(fileId, file, InputStreamContent(mimeType, stream))
update.fields = requiredRequestFields
return update.execute()
}
internal fun resolveFilesConflict(client: Drive, list: List<File>): File {
// Use newest file
val newest = list.maxBy { it.modifiedTime.value }!!
// Delete all others
val batch = client.batch()
val callback = SimpleJsonBatchCallback<Void>()
val files = client.files()
list.filterNot { it == newest }.forEach { files.delete(it.id).queue(batch, callback) }
batch.execute()
return newest
}
internal fun resolveFoldersConflict(client: Drive, list: List<File>): File {
return list.first()
}

View File

@ -1,6 +1,7 @@
package org.mariotaku.twidere.util.sync.google
import android.content.Context
import android.util.Log
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
import com.google.api.client.http.javanet.NetHttpTransport
import com.google.api.client.json.jackson2.JacksonFactory
@ -8,13 +9,11 @@ import com.google.api.services.drive.Drive
import nl.komponents.kovenant.task
import nl.komponents.kovenant.ui.failUi
import nl.komponents.kovenant.ui.successUi
import org.mariotaku.twidere.BuildConfig
import org.mariotaku.twidere.model.sync.GoogleDriveSyncProviderInfo
import org.mariotaku.twidere.util.TaskServiceRunner
import org.mariotaku.twidere.util.sync.ISyncAction
import org.mariotaku.twidere.util.sync.SyncTaskRunner
import org.mariotaku.twidere.util.sync.UserColorsSyncProcessor
import org.mariotaku.twidere.util.sync.UserNicknamesSyncProcessor
import org.mariotaku.twidere.util.sync.dropbox.DropboxPreferencesValuesSyncAction
import org.mariotaku.twidere.util.sync.*
import java.io.IOException
/**
@ -48,6 +47,9 @@ class GoogleDriveSyncTaskRunner(context: Context, val refreshToken: String) : Sy
}.successUi {
callback(true)
}.failUi {
if (BuildConfig.DEBUG) {
Log.w(LOGTAG_SYNC, "Sync $action failed", it)
}
callback(false)
}
return true

View File

@ -11,7 +11,7 @@ abstract internal class GoogleDriveUploadSession<in Data>(
val name: String,
val parentId: String,
val mimeType: String,
val files: Drive.Files
val drive: Drive
) : Closeable {
private var uploader: UploadUploader? = null
@ -25,7 +25,7 @@ abstract internal class GoogleDriveUploadSession<in Data>(
abstract fun Data.toInputStream(): InputStream
fun uploadData(data: Data): Boolean {
files.updateOrCreate(name, mimeType, parentId, stream = data.toInputStream(), fileConfig = {
drive.updateOrCreate(name, mimeType, parentId, stream = data.toInputStream(), fileConfig = {
it.modifiedTime = DateTime(localModifiedTime)
})
return true

View File

@ -0,0 +1,23 @@
package org.mariotaku.twidere.util.sync.google
import com.google.api.client.googleapis.batch.json.JsonBatchCallback
import com.google.api.client.googleapis.json.GoogleJsonError
import com.google.api.client.http.HttpHeaders
import java.io.IOException
/**
* Created by mariotaku on 1/22/17.
*/
internal class SimpleJsonBatchCallback<T> : JsonBatchCallback<T>() {
@Throws(IOException::class)
override fun onFailure(error: GoogleJsonError, headers: HttpHeaders) {
}
@Throws(IOException::class)
override fun onSuccess(result: T, headers: HttpHeaders) {
}
}

View File

@ -82,9 +82,11 @@ abstract class FileBasedDraftsSyncAction<RemoteFileInfo>(val context: Context) :
downloadRemoteInfoList.add(remoteDraft)
} else if (remoteDraft.draftTimestamp - localDraft.timestamp > 1000) {
// Local is older, update from remote
localDraft.remote_extras = remoteDraft.draftRemoteExtras
updateLocalInfoList[localDraft._id] = remoteDraft
} else if (localDraft.timestamp - remoteDraft.draftTimestamp > 1000) {
// Local is newer, upload local
localDraft.remote_extras = remoteDraft.draftRemoteExtras
uploadLocalList.add(localDraft)
}
}
@ -111,48 +113,58 @@ abstract class FileBasedDraftsSyncAction<RemoteFileInfo>(val context: Context) :
// Upload local items
if (BuildConfig.DEBUG && uploadLocalList.isNotEmpty()) {
val fileList = uploadLocalList.joinToString(",") { it.filename }
Log.d(LOGTAG_SYNC, "Uploading local drafts $fileList")
if (uploadLocalList.isNotEmpty()) {
if (BuildConfig.DEBUG) {
val fileList = uploadLocalList.joinToString(",") { it.filename }
Log.d(LOGTAG_SYNC, "Uploading local drafts $fileList")
}
uploadDrafts(uploadLocalList)
}
uploadDrafts(uploadLocalList)
// Download remote items
if (BuildConfig.DEBUG && downloadRemoteInfoList.isNotEmpty()) {
val fileList = downloadRemoteInfoList.joinToString(",") { it.draftFileName }
Log.d(LOGTAG_SYNC, "Downloading remote drafts $fileList")
if (downloadRemoteInfoList.isNotEmpty()) {
if (BuildConfig.DEBUG) {
val fileList = downloadRemoteInfoList.joinToString(",") { it.draftFileName }
Log.d(LOGTAG_SYNC, "Downloading remote drafts $fileList")
}
ContentResolverUtils.bulkInsert(context.contentResolver, Drafts.CONTENT_URI,
downloadDrafts(downloadRemoteInfoList).map { DraftValuesCreator.create(it) })
}
ContentResolverUtils.bulkInsert(context.contentResolver, Drafts.CONTENT_URI,
downloadDrafts(downloadRemoteInfoList).map { DraftValuesCreator.create(it) })
// Update local items
if (BuildConfig.DEBUG && updateLocalInfoList.size() > 0) {
val fileList = (0 until updateLocalInfoList.size()).joinToString(",") { updateLocalInfoList.valueAt(it).draftFileName }
Log.d(LOGTAG_SYNC, "Updating local drafts $fileList")
}
for (index in 0 until updateLocalInfoList.size()) {
val draft = Draft()
if (draft.loadFromRemote(updateLocalInfoList.valueAt(index))) {
val where = Expression.equalsArgs(Drafts._ID).sql
val whereArgs = arrayOf(updateLocalInfoList.keyAt(index).toString())
context.contentResolver.update(Drafts.CONTENT_URI, DraftValuesCreator.create(draft), where, whereArgs)
if (updateLocalInfoList.size() > 0) {
if (BuildConfig.DEBUG) {
val fileList = (0 until updateLocalInfoList.size()).joinToString(",") { updateLocalInfoList.valueAt(it).draftFileName }
Log.d(LOGTAG_SYNC, "Updating local drafts $fileList")
}
for (index in 0 until updateLocalInfoList.size()) {
val draft = Draft()
if (draft.loadFromRemote(updateLocalInfoList.valueAt(index))) {
val where = Expression.equalsArgs(Drafts._ID).sql
val whereArgs = arrayOf(updateLocalInfoList.keyAt(index).toString())
context.contentResolver.update(Drafts.CONTENT_URI, DraftValuesCreator.create(draft), where, whereArgs)
}
}
}
// Remove local items
if (BuildConfig.DEBUG && removeLocalIdsList.isNotEmpty()) {
val fileList = removeLocalIdsList.joinToString(",") { "$it.eml" }
Log.d(LOGTAG_SYNC, "Removing local drafts $fileList")
if (removeLocalIdsList.isNotEmpty()) {
if (BuildConfig.DEBUG) {
val fileList = removeLocalIdsList.joinToString(",") { "$it.eml" }
Log.d(LOGTAG_SYNC, "Removing local drafts $fileList")
}
ContentResolverUtils.bulkDelete(context.contentResolver, Drafts.CONTENT_URI,
Drafts.UNIQUE_ID, removeLocalIdsList, null)
}
ContentResolverUtils.bulkDelete(context.contentResolver, Drafts.CONTENT_URI,
Drafts.UNIQUE_ID, removeLocalIdsList, null)
// Remove remote items
if (BuildConfig.DEBUG && removeRemoteInfoList.isNotEmpty()) {
val fileList = removeRemoteInfoList.joinToString(",") { it.draftFileName }
Log.d(LOGTAG_SYNC, "Removing remote drafts $fileList")
if (removeRemoteInfoList.isNotEmpty()) {
if (BuildConfig.DEBUG) {
val fileList = removeRemoteInfoList.joinToString(",") { it.draftFileName }
Log.d(LOGTAG_SYNC, "Removing remote drafts $fileList")
}
removeDrafts(removeRemoteInfoList)
}
removeDrafts(removeRemoteInfoList)
snapshotsListFile.writer().use { writer ->
val cur = context.contentResolver.query(Drafts.CONTENT_URI, Drafts.COLUMNS, null, null, null)!!
@ -216,6 +228,7 @@ abstract class FileBasedDraftsSyncAction<RemoteFileInfo>(val context: Context) :
abstract val RemoteFileInfo.draftFileName: String
abstract val RemoteFileInfo.draftTimestamp: Long
open val RemoteFileInfo.draftRemoteExtras: String? get() = null
@Throws(IOException::class)
open fun setup(): Boolean = true

View File

@ -42,7 +42,7 @@ abstract class SingleFileBasedDataSyncAction<Data, SnapshotStore, DownloadSessio
}
} catch (e: FileNotFoundException) {
if (BuildConfig.DEBUG) {
Log.d(LOGTAG_SYNC, "Remote $whatData doesn't exists, will upload new one")
Log.d(LOGTAG_SYNC, "Remote $whatData doesn't exist, will upload new one")
}
shouldCreateRemote = true
}