Remember the ETag header for sources and use it to detect sources which are already up to date

Implements #147

app/schemas/com.frostnerd.smokescreen.database.AppDatabase/10.json

@@ -2,7 +2,7 @@
   "formatVersion": 1,
   "database": {
     "version": 10,
-    "identityHash": "daac9efbbf9d89b9aa10f7ad54ac736a",
+    "identityHash": "5d29ef2a948e6e550e83fcef5932253d",
     "entities": [
       {
         "tableName": "CachedResponse",
@@ -194,7 +194,7 @@
       },
       {
         "tableName": "HostSource",
-        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `enabled` INTEGER NOT NULL, `ruleCount` INTEGER, `name` TEXT NOT NULL, `source` TEXT NOT NULL, `whitelistSource` INTEGER NOT NULL)",
+        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `enabled` INTEGER NOT NULL, `ruleCount` INTEGER, `checksum` TEXT, `name` TEXT NOT NULL, `source` TEXT NOT NULL, `whitelistSource` INTEGER NOT NULL)",
         "fields": [
           {
             "fieldPath": "id",
@@ -214,6 +214,12 @@
             "affinity": "INTEGER",
             "notNull": false
           },
+          {
+            "fieldPath": "checksum",
+            "columnName": "checksum",
+            "affinity": "TEXT",
+            "notNull": false
+          },
           {
             "fieldPath": "name",
             "columnName": "name",
@@ -246,7 +252,7 @@
     "views": [],
     "setupQueries": [
       "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
-      "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'daac9efbbf9d89b9aa10f7ad54ac736a')"
+      "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '5d29ef2a948e6e550e83fcef5932253d')"
     ]
   }
 }
\ No newline at end of file

app/src/main/java/com/frostnerd/smokescreen/database/Extensions.kt

@@ -93,6 +93,7 @@ val MIGRATION_8_9 = migration(8, 9) {
 val MIGRATION_9_10 = migration(9, 10) {
     Logger.logIfOpen("DB_MIGRATION", "Migrating from 9 to 10")
     it.execSQL("ALTER TABLE `DnsRule` ADD COLUMN `isWildcard` INTEGER NOT NULL DEFAULT 0")
+    it.execSQL("ALTER TABLE `HostSource` ADD COLUMN `checksum` TEXT DEFAULT NULL")
     Logger.logIfOpen("DB_MIGRATION", "Migration from 9 to 10 completed")
 }
 

app/src/main/java/com/frostnerd/smokescreen/database/dao/DnsRuleDao.kt

@@ -45,9 +45,12 @@ interface DnsRuleDao {
     @Query("DELETE FROM DnsRule WHERE importedFrom IS NULL")
     fun deleteAllUserRules()
 
-    @Query("UPDATE DnsRule SET stagingType=1 WHERE importedFrom IS NOT NULL")
+    @Query("UPDATE DnsRule SET stagingType=1 WHERE importedFrom IS NOT NULL AND stagingType=0")
     fun markNonUserRulesForDeletion()
 
+    @Query("UPDATE DnsRule SET stagingType=0 WHERE importedFrom=:hostSourceId AND stagingType=1")
+    fun unstageRulesOfSource(hostSourceId:Long)
+
     @Query("DELETE FROM DnsRule WHERE stagingType=1")
     fun deleteMarkedRules()
 

app/src/main/java/com/frostnerd/smokescreen/database/dao/HostSourceDao.kt

@@ -47,4 +47,7 @@ interface HostSourceDao {
 
     @Query("SELECT COUNT(*) FROM HostSource WHERE enabled > 0")
     fun getEnabledCount(): Long
+
+    @Query("UPDATE HostSource SET checksum=NULL WHERE checksum IS NOT NULL AND enabled<1")
+    fun removeChecksumForDisabled()
 }
\ No newline at end of file

app/src/main/java/com/frostnerd/smokescreen/database/entities/HostSource.kt

@@ -35,4 +35,7 @@ data class HostSource(
     var enabled: Boolean = true
     var ruleCount:Int? = null
 
+    // Content of the ETag header for supported rule sources
+    // Null if file-based or unknown (e.g. not requested yet)
+    var checksum:String? = null
 }
\ No newline at end of file

app/src/main/java/com/frostnerd/smokescreen/service/RuleImportService.kt

@@ -166,6 +166,7 @@ class RuleImportService : IntentService("RuleImportService") {
         dnsRuleDao.deleteStagedRules()
         var count = 0
         val maxCount = getDatabase().hostSourceDao().getEnabledCount()
+        val newChecksums = mutableMapOf<HostSource, String>()
         getDatabase().hostSourceDao().getAllEnabled().forEach {
             log("Importing HostSource $it")
             if (!isAborted) {
@@ -189,15 +190,31 @@ class RuleImportService : IntentService("RuleImportService") {
                     var response: Response? = null
                     try {
                         val request = Request.Builder().url(it.source)
+                        if(it.checksum != null) request.header("If-None-Match", it.checksum!!)
                         response = httpClient.newCall(request.build()).execute()
-                        if (response.isSuccessful) {
-                            processLines(it, response.body!!.byteStream())
-                        } else {
-                            log("Downloading resource of $it failed.")
+                        val localDataIsRecent = response.code == 304 || it.checksum!= null && response.headers.find {
+                            it.first.equals("etag", true)
+                        }?.second == it.checksum
+                        when {
+                            response.isSuccessful && !localDataIsRecent -> {
+                                response.headers.find {
+                                    it.first.equals("etag", true)
+                                }?.second?.apply {
+                                    newChecksums[it] = this
+                                }
+                                processLines(it, response.body!!.byteStream())
+                            }
+                            response.code == 304 || localDataIsRecent -> {
+                                log("Host source ${it.name} hasn't changed, not updating.")
+                                ruleCount += it.ruleCount ?: 0
+                                dnsRuleDao.unstageRulesOfSource(it.id)
+                                log("Unstaged rules for ${it.name}")
+                            }
+                            else -> log("Downloading resource of ${it.name} failed.")
                         }
                     } catch (ex: java.lang.Exception) {
                         ex.printStackTrace()
-                        log("Downloading resource of $it failed ($ex)")
+                        log("Downloading resource of ${it.name} failed ($ex)")
                     } finally {
                         response?.body?.close()
                     }
@@ -217,6 +234,12 @@ class RuleImportService : IntentService("RuleImportService") {
             dnsRuleDao.deleteStagedRules()
             log("Recreating database indices")
             getDatabase().recreateDnsRuleIndizes()
+            log("Updating Etag values for sources")
+            newChecksums.forEach { (source, etag) ->
+                source.checksum = etag
+                getDatabase().hostSourceDao().update(source)
+            }
+            getDatabase().hostSourceDao().removeChecksumForDisabled()
             log("Done.")
             showSuccessNotification()
         } else {