isoron/loop
1
1
Fork 0
mirror of https://github.com/iSoron/uhabits.git synced 2025-12-06 09:08:52 -06:00
Code Issues Packages Projects Releases Wiki Activity

Minor fixes to sync protocol

Browse Source
This commit is contained in:
Alinson S. Xavier
2020-11-28 11:32:28 -06:00
parent 1787c0e74e
commit 9c0951ae58
13 changed files with 135 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
android/uhabits-android/src/androidTest/java/org/isoron/uhabits/sync/RemoteSyncServerTest.kt
View File

@@ -19,6 +19,7 @@
package org.isoron.uhabits.sync
import androidx.test.filters.*
import com.fasterxml.jackson.databind.*
import io.ktor.client.*
import io.ktor.client.engine.mock.*
@@ -29,6 +30,7 @@ import junit.framework.Assert.*
import kotlinx.coroutines.*
import org.junit.*
@MediumTest
class RemoteSyncServerTest {
private val mapper = ObjectMapper()
@@ -53,7 +55,7 @@ class RemoteSyncServerTest {
@Test
fun when_get_data_version_succeeds_should_return_version() = runBlocking {
server("/ABC/version") {
server("/db/ABC/version") {
respondWithJson(GetDataVersionResponse(5))
}.apply {
assertEquals(5, getDataVersion("ABC"))
@@ -63,7 +65,7 @@ class RemoteSyncServerTest {
@Test(expected = ServiceUnavailable::class)
fun when_get_data_version_with_server_error_should_raise_exception() = runBlocking {
server("/ABC/version") {
server("/db/ABC/version") {
respondError(HttpStatusCode.InternalServerError)
}.apply {
getDataVersion("ABC")
@@ -73,7 +75,7 @@ class RemoteSyncServerTest {
@Test(expected = KeyNotFoundException::class)
fun when_get_data_version_with_invalid_key_should_raise_exception() = runBlocking {
server("/ABC/version") {
server("/db/ABC/version") {
respondError(HttpStatusCode.NotFound)
}.apply {
getDataVersion("ABC")
@@ -83,7 +85,7 @@ class RemoteSyncServerTest {
@Test
fun when_get_data_succeeds_should_return_data() = runBlocking {
server("/ABC") {
server("/db/ABC") {
respondWithJson(data)
}.apply {
assertEquals(data, getData("ABC"))
@@ -93,7 +95,7 @@ class RemoteSyncServerTest {
@Test(expected = KeyNotFoundException::class)
fun when_get_data_with_invalid_key_should_raise_exception() = runBlocking {
server("/ABC") {
server("/db/ABC") {
respondError(HttpStatusCode.NotFound)
}.apply {
getData("ABC")
@@ -103,7 +105,7 @@ class RemoteSyncServerTest {
@Test
fun when_put_succeeds_should_not_raise_exceptions() = runBlocking {
server("/ABC") {
server("/db/ABC") {
respondOk()
}.apply {
put("ABC", data)

11
android/uhabits-android/src/main/java/org/isoron/uhabits/sync/RemoteSyncServer.kt
View File

@@ -19,6 +19,7 @@
package org.isoron.uhabits.sync
import android.util.*
import io.ktor.client.*
import io.ktor.client.engine.android.*
import io.ktor.client.features.*
@@ -26,9 +27,6 @@ import io.ktor.client.features.json.*
import io.ktor.client.request.*
import kotlinx.coroutines.*
data class RegisterReponse(val key: String)
data class GetDataVersionResponse(val version: Long)
class RemoteSyncServer(
private val baseURL: String = "https://sync.loophabits.org",
private val httpClient: HttpClient = HttpClient(Android) {
@@ -54,7 +52,10 @@ class RemoteSyncServer(
} catch (e: ServerResponseException) {
throw ServiceUnavailable()
} catch (e: ClientRequestException) {
throw KeyNotFoundException()
Log.w("RemoteSyncServer", "ClientRequestException", e)
if(e.message!!.contains("409")) throw EditConflictException()
if(e.message!!.contains("404")) throw KeyNotFoundException()
throw e
}
}
@@ -65,6 +66,7 @@ class RemoteSyncServer(
} catch (e: ServerResponseException) {
throw ServiceUnavailable()
} catch (e: ClientRequestException) {
Log.w("RemoteSyncServer", "ClientRequestException", e)
throw KeyNotFoundException()
}
}
@@ -76,6 +78,7 @@ class RemoteSyncServer(
} catch(e: ServerResponseException) {
throw ServiceUnavailable()
} catch (e: ClientRequestException) {
Log.w("RemoteSyncServer", "ClientRequestException", e)
throw KeyNotFoundException()
}
}

4
android/uhabits-android/src/main/java/org/isoron/uhabits/sync/SyncData.kt
View File

@@ -23,3 +23,7 @@ data class SyncData(
val version: Long,
val content: String
)
data class RegisterReponse(val key: String)
data class GetDataVersionResponse(val version: Long)

66
android/uhabits-android/src/main/java/org/isoron/uhabits/sync/SyncManager.kt
View File

@@ -30,6 +30,7 @@ import org.isoron.uhabits.core.tasks.*
import org.isoron.uhabits.tasks.*
import org.isoron.uhabits.utils.*
import java.io.*
import java.lang.RuntimeException
import javax.inject.*
@AppScope
@@ -43,7 +44,7 @@ class SyncManager @Inject constructor(
private val server = RemoteSyncServer()
private val tmpFile = File.createTempFile("import", "", context.externalCacheDir)
private var currVersion = 0L
private var currVersion = 1L
private var dirty = true
private lateinit var encryptionKey: EncryptionKey
@@ -72,35 +73,53 @@ class SyncManager @Inject constructor(
}
}
private suspend fun push() {
if (!dirty) {
Log.i("SyncManager", "Database not dirty. Skipping upload.")
return
private suspend fun push(depth: Int = 0) {
if(depth >= 5) {
throw RuntimeException()
}
if (dirty) {
Log.i("SyncManager", "Encrypting local database...")
val db = DatabaseUtils.getDatabaseFile(context)
val encryptedDB = db.encryptToString(encryptionKey)
val size = encryptedDB.length / 1024
Log.i("SyncManager", "Pushing local database (version $currVersion, $size KB)")
try {
server.put(preferences.syncKey, SyncData(currVersion, encryptedDB))
dirty = false
} catch (e: EditConflictException) {
Log.i("SyncManager", "Sync conflict detected while pushing.")
setCurrentVersion(0)
pull()
push(depth = depth + 1)
}
} else {
Log.i("SyncManager", "Local database not modified. Skipping push.")
}
Log.i("SyncManager", "Encrypting database...")
val db = DatabaseUtils.getDatabaseFile(context)
val encryptedDB = db.encryptToString(encryptionKey)
Log.i("SyncManager", "Uploading database (version ${currVersion}, ${encryptedDB.length / 1024} KB)")
server.put(preferences.syncKey, SyncData(currVersion, encryptedDB))
dirty = false
}
private suspend fun pull() {
Log.i("SyncManager", "Fetching database from server...")
val data = server.getData(preferences.syncKey)
Log.i("SyncManager", "Fetched database (version ${data.version}, ${data.content.length / 1024} KB)")
if (data.version == 0L) {
Log.i("SyncManager", "Initial upload detected. Marking db as dirty.")
dirty = true
}
if (data.version <= currVersion) {
Log.i("SyncManager", "Local version is up-to-date. Skipping merge.")
Log.i("SyncManager", "Querying remote database version...")
val remoteVersion = server.getDataVersion(syncKey)
Log.i("SyncManager", "Remote database has version $remoteVersion")
if (remoteVersion <= currVersion) {
Log.i("SyncManager", "Local database is up-to-date. Skipping merge.")
} else {
Log.i("SyncManager", "Decrypting and merging with local changes...")
Log.i("SyncManager", "Pulling remote database...")
val data = server.getData(syncKey)
val size = data.content.length / 1024
Log.i("SyncManager", "Pulled remote database (version ${data.version}, $size KB)")
Log.i("SyncManager", "Decrypting remote database and merging with local changes...")
data.content.decryptToFile(encryptionKey, tmpFile)
taskRunner.execute(importDataTaskFactory.create(tmpFile) { tmpFile.delete() })
dirty = true
setCurrentVersion(data.version + 1)
}
currVersion = data.version + 1
}
private fun setCurrentVersion(v: Long) {
currVersion = v
Log.i("SyncManager", "Setting local database version to $currVersion")
}
suspend fun onResume() {
@@ -118,6 +137,9 @@ class SyncManager @Inject constructor(
}
override fun onCommandExecuted(command: Command?, refreshKey: Long?) {
if (!dirty) {
setCurrentVersion(currVersion + 1)
}
dirty = true
}
}

26
server/src/org/isoron/uhabits/sync/AbstractSyncServer.kt
View File

@@ -24,11 +24,10 @@ interface AbstractSyncServer {
* Generates and returns a new sync key, which can be used to store and retrive
* data.
*
* @throws RegistrationUnavailableException If key cannot be generated at this
* time, for example, due to insufficient server resources or temporary
* maintenance.
* @throws ServiceUnavailable If key cannot be generated at this time, for example,
* due to insufficient server resources, temporary server maintenance or network problems.
*/
fun register(): String
suspend fun register(): String
/**
* Replaces data for a given sync key.
@@ -36,13 +35,26 @@ interface AbstractSyncServer {
* @throws KeyNotFoundException If key is not found
* @throws EditConflictException If the version of the data provided is not
* exactly the current data version plus one.
* @throws ServiceUnavailable If data cannot be put at this time, for example, due
* to insufficient server resources or network problems.
*/
fun put(key: String, newData: SyncData)
suspend fun put(key: String, newData: SyncData)
/**
* Returns data for a given sync key.
*
* @throws KeyNotFoundException If key is not found
* @throws ServiceUnavailable If data cannot be retrieved at this time, for example, due
* to insufficient server resources or network problems.
*/
fun get(key: String): SyncData
}
suspend fun getData(key: String): SyncData
/**
* Returns the current data version for the given key
*
* @throws KeyNotFoundException If key is not found
* @throws ServiceUnavailable If data cannot be retrieved at this time, for example, due
* to insufficient server resources or network problems.
*/
suspend fun getDataVersion(key: String): Long
}

15
server/src/org/isoron/uhabits/sync/MemorySyncServer.kt
View File

@@ -28,7 +28,7 @@ import kotlin.streams.*
class MemorySyncServer : AbstractSyncServer {
private val db = mutableMapOf<String, SyncData>()
override fun register(): String {
override suspend fun register(): String {
synchronized(db) {
val key = generateKey()
db[key] = SyncData(0, "")
@@ -36,7 +36,7 @@ class MemorySyncServer : AbstractSyncServer {
}
}
override fun put(key: String, newData: SyncData) {
override suspend fun put(key: String, newData: SyncData) {
synchronized(db) {
if (!db.containsKey(key)) {
throw KeyNotFoundException()
@@ -49,7 +49,7 @@ class MemorySyncServer : AbstractSyncServer {
}
}
override fun get(key: String): SyncData {
override suspend fun getData(key: String): SyncData {
synchronized(db) {
if (!db.containsKey(key)) {
throw KeyNotFoundException()
@@ -58,6 +58,15 @@ class MemorySyncServer : AbstractSyncServer {
}
}
override suspend fun getDataVersion(key: String): Long {
synchronized(db) {
if (!db.containsKey(key)) {
throw KeyNotFoundException()
}
return db.getValue(key).version
}
}
private fun generateKey(): String {
val chars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
while (true) {

9
server/src/org/isoron/uhabits/sync/SyncData.kt
View File

@@ -22,9 +22,14 @@ package org.isoron.uhabits.sync
import com.fasterxml.jackson.databind.*
data class SyncData(
val version: Int,
val content: String,
val version: Long,
val content: String
)
data class RegisterReponse(val key: String)
data class GetDataVersionResponse(val version: Long)
val defaultMapper = ObjectMapper()
fun SyncData.toJson(): String = defaultMapper.writeValueAsString(this)
fun GetDataVersionResponse.toJson(): String = defaultMapper.writeValueAsString(this)

5
server/src/org/isoron/uhabits/sync/SyncException.kt
View File

@@ -19,13 +19,10 @@
package org.isoron.uhabits.sync
/**
* Generic class for all exceptions thrown by SyncServer.
*/
open class SyncException: RuntimeException()
class KeyNotFoundException: SyncException()
class RegistrationUnavailableException: SyncException()
class ServiceUnavailable: SyncException()
class EditConflictException: SyncException()

4
server/src/org/isoron/uhabits/sync/app/RegistrationModule.kt
View File

@@ -29,8 +29,8 @@ fun Routing.registration(app: SyncApplication) {
post("/register") {
try {
val key = app.server.register()
call.respond(HttpStatusCode.OK, mapOf("key" to key))
} catch (e: RegistrationUnavailableException) {
call.respond(HttpStatusCode.OK, RegisterReponse(key))
} catch (e: ServiceUnavailable) {
call.respond(HttpStatusCode.ServiceUnavailable)
}
}

9
server/src/org/isoron/uhabits/sync/app/StorageModule.kt
View File

@@ -31,7 +31,7 @@ fun Routing.storage(app: SyncApplication) {
get {
val key = call.parameters["key"]!!
try {
val data = app.server.get(key)
val data = app.server.getData(key)
call.respond(HttpStatusCode.OK, data)
} catch(e: KeyNotFoundException) {
call.respond(HttpStatusCode.NotFound)
@@ -46,15 +46,14 @@ fun Routing.storage(app: SyncApplication) {
} catch (e: KeyNotFoundException) {
call.respond(HttpStatusCode.NotFound)
} catch (e: EditConflictException) {
val currData = app.server.get(key)
call.respond(HttpStatusCode.Conflict, currData)
call.respond(HttpStatusCode.Conflict)
}
}
get("version") {
val key = call.parameters["key"]!!
try {
val data = app.server.get(key)
call.respond(HttpStatusCode.OK, data.version)
val version = app.server.getDataVersion(key)
call.respond(HttpStatusCode.OK, GetDataVersionResponse(version))
} catch(e: KeyNotFoundException) {
call.respond(HttpStatusCode.NotFound)
}

13
server/test/org/isoron/uhabits/sync/MemorySyncServerTest.kt
View File

@@ -19,33 +19,34 @@
package org.isoron.uhabits.sync
import kotlinx.coroutines.*
import org.junit.Test
import kotlin.test.*
class MemorySyncServerTest {
private val server = MemorySyncServer()
private val key = server.register()
private val key = runBlocking { server.register() }
@Test
fun testUsage() {
fun testUsage(): Unit = runBlocking {
val data0 = SyncData(0, "")
assertEquals(server.get(key), data0)
assertEquals(server.getData(key), data0)
val data1 = SyncData(1, "Hello world")
server.put(key, data1)
assertEquals(server.get(key), data1)
assertEquals(server.getData(key), data1)
val data2 = SyncData(2, "Hello new world")
server.put(key, data2)
assertEquals(server.get(key), data2)
assertEquals(server.getData(key), data2)
assertFailsWith<EditConflictException> {
server.put(key, data2)
}
assertFailsWith<KeyNotFoundException> {
server.get("INVALID")
server.getData("INVALID")
}
assertFailsWith<KeyNotFoundException> {

7
server/test/org/isoron/uhabits/sync/app/RegistrationModuleTest.kt
View File

@@ -21,6 +21,7 @@ package org.isoron.uhabits.sync.app
import io.ktor.http.*
import io.ktor.server.testing.*
import kotlinx.coroutines.*
import org.isoron.uhabits.sync.*
import org.junit.Test
import org.mockito.*
@@ -29,7 +30,7 @@ import kotlin.test.*
class RegistrationModuleTest : BaseApplicationTest() {
@Test
fun `when register succeeds should return generated key`() {
fun `when register succeeds should return generated key`():Unit = runBlocking {
`when`(server.register()).thenReturn("ABCDEF")
withTestApplication(app()) {
val call = handleRequest(HttpMethod.Post, "/register")
@@ -39,8 +40,8 @@ class RegistrationModuleTest : BaseApplicationTest() {
}
@Test
fun `when registration is unavailable should return 503`() {
`when`(server.register()).thenThrow(RegistrationUnavailableException())
fun `when registration is unavailable should return 503`():Unit = runBlocking {
`when`(server.register()).thenThrow(ServiceUnavailable())
withTestApplication(app()) {
val call = handleRequest(HttpMethod.Post, "/register")
assertEquals(HttpStatusCode.ServiceUnavailable, call.response.status())

30
server/test/org/isoron/uhabits/sync/app/StorageModuleTest.kt
View File

@@ -21,6 +21,7 @@ package org.isoron.uhabits.sync.app
import io.ktor.http.*
import io.ktor.server.testing.*
import kotlinx.coroutines.*
import org.isoron.uhabits.sync.*
import org.junit.Test
import org.mockito.Mockito.*
@@ -31,8 +32,8 @@ class StorageModuleTest : BaseApplicationTest() {
private val data2 = SyncData(2, "Hello new world")
@Test
fun `when get succeeds should return data`() {
`when`(server.get("k1")).thenReturn(data1)
fun `when get succeeds should return data`(): Unit = runBlocking {
`when`(server.getData("k1")).thenReturn(data1)
withTestApplication(app()) {
handleGet("/db/k1").apply {
assertEquals(HttpStatusCode.OK, response.status())
@@ -42,19 +43,19 @@ class StorageModuleTest : BaseApplicationTest() {
}
@Test
fun `when get version succeeds should return version`() {
`when`(server.get("k1")).thenReturn(data1)
fun `when get version succeeds should return version`(): Unit = runBlocking {
`when`(server.getDataVersion("k1")).thenReturn(30)
withTestApplication(app()) {
handleGet("/db/k1/version").apply {
assertEquals(HttpStatusCode.OK, response.status())
assertEquals("1", response.content)
assertEquals(GetDataVersionResponse(30).toJson(), response.content)
}
}
}
@Test
fun `when get with invalid key should return 404`() {
`when`(server.get("k1")).thenThrow(KeyNotFoundException())
fun `when get with invalid key should return 404`(): Unit = runBlocking {
`when`(server.getData("k1")).thenThrow(KeyNotFoundException())
withTestApplication(app()) {
handleGet("/db/k1").apply {
assertEquals(HttpStatusCode.NotFound, response.status())
@@ -64,17 +65,19 @@ class StorageModuleTest : BaseApplicationTest() {
@Test
fun `when put succeeds should return OK`() {
fun `when put succeeds should return OK`(): Unit = runBlocking {
withTestApplication(app()) {
handlePut("/db/k1", data1).apply {
assertEquals(HttpStatusCode.OK, response.status())
verify(server).put("k1", data1)
runBlocking {
assertEquals(HttpStatusCode.OK, response.status())
verify(server).put("k1", data1)
}
}
}
}
@Test
fun `when put with invalid key should return 404`() {
fun `when put with invalid key should return 404`(): Unit = runBlocking {
`when`(server.put("k1", data1)).thenThrow(KeyNotFoundException())
withTestApplication(app()) {
handlePut("/db/k1", data1).apply {
@@ -84,13 +87,12 @@ class StorageModuleTest : BaseApplicationTest() {
}
@Test
fun `when put with invalid version should return 409 and current data`() {
fun `when put with invalid version should return 409 and current data`(): Unit = runBlocking {
`when`(server.put("k1", data1)).thenThrow(EditConflictException())
`when`(server.get("k1")).thenReturn(data2)
`when`(server.getData("k1")).thenReturn(data2)
withTestApplication(app()) {
handlePut("/db/k1", data1).apply {
assertEquals(HttpStatusCode.Conflict, response.status())
assertEquals(data2.toJson(), response.content)
}
}
}