Commit 6efe2e2e authored by Daniel Wolf's avatar Daniel Wolf
Browse files

Remember the ETag header for sources and use it to detect sources which are already up to date

Implements #147
parent 2153ee9d
......@@ -2,7 +2,7 @@
"formatVersion": 1,
"database": {
"version": 10,
"identityHash": "daac9efbbf9d89b9aa10f7ad54ac736a",
"identityHash": "5d29ef2a948e6e550e83fcef5932253d",
"entities": [
{
"tableName": "CachedResponse",
......@@ -194,7 +194,7 @@
},
{
"tableName": "HostSource",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `enabled` INTEGER NOT NULL, `ruleCount` INTEGER, `name` TEXT NOT NULL, `source` TEXT NOT NULL, `whitelistSource` INTEGER NOT NULL)",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `enabled` INTEGER NOT NULL, `ruleCount` INTEGER, `checksum` TEXT, `name` TEXT NOT NULL, `source` TEXT NOT NULL, `whitelistSource` INTEGER NOT NULL)",
"fields": [
{
"fieldPath": "id",
......@@ -214,6 +214,12 @@
"affinity": "INTEGER",
"notNull": false
},
{
"fieldPath": "checksum",
"columnName": "checksum",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "name",
"columnName": "name",
......@@ -246,7 +252,7 @@
"views": [],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'daac9efbbf9d89b9aa10f7ad54ac736a')"
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '5d29ef2a948e6e550e83fcef5932253d')"
]
}
}
\ No newline at end of file
......@@ -93,6 +93,7 @@ val MIGRATION_8_9 = migration(8, 9) {
val MIGRATION_9_10 = migration(9, 10) {
Logger.logIfOpen("DB_MIGRATION", "Migrating from 9 to 10")
it.execSQL("ALTER TABLE `DnsRule` ADD COLUMN `isWildcard` INTEGER NOT NULL DEFAULT 0")
it.execSQL("ALTER TABLE `HostSource` ADD COLUMN `checksum` TEXT DEFAULT NULL")
Logger.logIfOpen("DB_MIGRATION", "Migration from 9 to 10 completed")
}
......
......@@ -45,9 +45,12 @@ interface DnsRuleDao {
@Query("DELETE FROM DnsRule WHERE importedFrom IS NULL")
fun deleteAllUserRules()
@Query("UPDATE DnsRule SET stagingType=1 WHERE importedFrom IS NOT NULL")
@Query("UPDATE DnsRule SET stagingType=1 WHERE importedFrom IS NOT NULL AND stagingType=0")
fun markNonUserRulesForDeletion()
@Query("UPDATE DnsRule SET stagingType=0 WHERE importedFrom=:hostSourceId AND stagingType=1")
fun unstageRulesOfSource(hostSourceId:Long)
@Query("DELETE FROM DnsRule WHERE stagingType=1")
fun deleteMarkedRules()
......
......@@ -47,4 +47,7 @@ interface HostSourceDao {
@Query("SELECT COUNT(*) FROM HostSource WHERE enabled > 0")
fun getEnabledCount(): Long
@Query("UPDATE HostSource SET checksum=NULL WHERE checksum IS NOT NULL AND enabled<1")
fun removeChecksumForDisabled()
}
\ No newline at end of file
......@@ -35,4 +35,7 @@ data class HostSource(
var enabled: Boolean = true
var ruleCount:Int? = null
// Content of the ETag header for supported rule sources
// Null if file-based or unknown (e.g. not requested yet)
var checksum:String? = null
}
\ No newline at end of file
......@@ -166,6 +166,7 @@ class RuleImportService : IntentService("RuleImportService") {
dnsRuleDao.deleteStagedRules()
var count = 0
val maxCount = getDatabase().hostSourceDao().getEnabledCount()
val newChecksums = mutableMapOf<HostSource, String>()
getDatabase().hostSourceDao().getAllEnabled().forEach {
log("Importing HostSource $it")
if (!isAborted) {
......@@ -189,15 +190,31 @@ class RuleImportService : IntentService("RuleImportService") {
var response: Response? = null
try {
val request = Request.Builder().url(it.source)
if(it.checksum != null) request.header("If-None-Match", it.checksum!!)
response = httpClient.newCall(request.build()).execute()
if (response.isSuccessful) {
processLines(it, response.body!!.byteStream())
} else {
log("Downloading resource of $it failed.")
val localDataIsRecent = response.code == 304 || it.checksum!= null && response.headers.find {
it.first.equals("etag", true)
}?.second == it.checksum
when {
response.isSuccessful && !localDataIsRecent -> {
response.headers.find {
it.first.equals("etag", true)
}?.second?.apply {
newChecksums[it] = this
}
processLines(it, response.body!!.byteStream())
}
response.code == 304 || localDataIsRecent -> {
log("Host source ${it.name} hasn't changed, not updating.")
ruleCount += it.ruleCount ?: 0
dnsRuleDao.unstageRulesOfSource(it.id)
log("Unstaged rules for ${it.name}")
}
else -> log("Downloading resource of ${it.name} failed.")
}
} catch (ex: java.lang.Exception) {
ex.printStackTrace()
log("Downloading resource of $it failed ($ex)")
log("Downloading resource of ${it.name} failed ($ex)")
} finally {
response?.body?.close()
}
......@@ -217,6 +234,12 @@ class RuleImportService : IntentService("RuleImportService") {
dnsRuleDao.deleteStagedRules()
log("Recreating database indices")
getDatabase().recreateDnsRuleIndizes()
log("Updating Etag values for sources")
newChecksums.forEach { (source, etag) ->
source.checksum = etag
getDatabase().hostSourceDao().update(source)
}
getDatabase().hostSourceDao().removeChecksumForDisabled()
log("Done.")
showSuccessNotification()
} else {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment