Commit e646f26c authored by Daniel Wolf's avatar Daniel Wolf
Browse files

Add filter to query log

This commit also adds various optimizations to the DnsQuery entity:
 - responses is not a MutableList anymore. This allows emptyList() to be used, saving memory.
 (- because of the prior point the query listener does not need to convert the responses to a MutableList from an immutable list, saving memory)
 - parsedResponses is now lazy. Before it was calculated for every row in the query log (every time it was scrolled to)
 - Added isHostBlockedByDnsServer, which makes it obsolete for the query log to calculate this flag for every row.
 - The StringListConverter now doesn't call gson for empty lists, saving time
 -

Closes #261
parent 8e8b21a1
{
"formatVersion": 1,
"database": {
"version": 12,
"identityHash": "4b8a3541fc27086145381f661dbb9a39",
"entities": [
{
"tableName": "CachedResponse",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`dnsName` TEXT NOT NULL, `type` INTEGER NOT NULL, `records` TEXT NOT NULL, PRIMARY KEY(`dnsName`, `type`))",
"fields": [
{
"fieldPath": "dnsName",
"columnName": "dnsName",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "type",
"columnName": "type",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "records",
"columnName": "records",
"affinity": "TEXT",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"dnsName",
"type"
],
"autoGenerate": false
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "DnsQuery",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `type` INTEGER NOT NULL, `name` TEXT NOT NULL, `askedServer` TEXT, `responseSource` TEXT NOT NULL, `questionTime` INTEGER NOT NULL, `responseTime` INTEGER NOT NULL, `responses` TEXT NOT NULL, `isHostBlockedByDnsServer` INTEGER NOT NULL)",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "type",
"columnName": "type",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "askedServer",
"columnName": "askedServer",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "responseSource",
"columnName": "responseSource",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "questionTime",
"columnName": "questionTime",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "responseTime",
"columnName": "responseTime",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "responses",
"columnName": "responses",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "isHostBlockedByDnsServer",
"columnName": "isHostBlockedByDnsServer",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"id"
],
"autoGenerate": true
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "DnsRule",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `stagingType` INTEGER NOT NULL, `type` INTEGER NOT NULL, `host` TEXT NOT NULL, `target` TEXT NOT NULL, `ipv6Target` TEXT, `importedFrom` INTEGER, `isWildcard` INTEGER NOT NULL, FOREIGN KEY(`importedFrom`) REFERENCES `HostSource`(`id`) ON UPDATE NO ACTION ON DELETE NO ACTION )",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "stagingType",
"columnName": "stagingType",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "type",
"columnName": "type",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "host",
"columnName": "host",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "target",
"columnName": "target",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "ipv6Target",
"columnName": "ipv6Target",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "importedFrom",
"columnName": "importedFrom",
"affinity": "INTEGER",
"notNull": false
},
{
"fieldPath": "isWildcard",
"columnName": "isWildcard",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"id"
],
"autoGenerate": true
},
"indices": [
{
"name": "index_DnsRule_importedFrom",
"unique": false,
"columnNames": [
"importedFrom"
],
"createSql": "CREATE INDEX IF NOT EXISTS `index_DnsRule_importedFrom` ON `${TABLE_NAME}` (`importedFrom`)"
},
{
"name": "index_DnsRule_host_type_stagingType",
"unique": true,
"columnNames": [
"host",
"type",
"stagingType"
],
"createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_DnsRule_host_type_stagingType` ON `${TABLE_NAME}` (`host`, `type`, `stagingType`)"
}
],
"foreignKeys": [
{
"table": "HostSource",
"onDelete": "NO ACTION",
"onUpdate": "NO ACTION",
"columns": [
"importedFrom"
],
"referencedColumns": [
"id"
]
}
]
},
{
"tableName": "HostSource",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `enabled` INTEGER NOT NULL, `ruleCount` INTEGER, `checksum` TEXT, `name` TEXT NOT NULL, `source` TEXT NOT NULL, `whitelistSource` INTEGER NOT NULL)",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "enabled",
"columnName": "enabled",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "ruleCount",
"columnName": "ruleCount",
"affinity": "INTEGER",
"notNull": false
},
{
"fieldPath": "checksum",
"columnName": "checksum",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "source",
"columnName": "source",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "whitelistSource",
"columnName": "whitelistSource",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"id"
],
"autoGenerate": true
},
"indices": [],
"foreignKeys": []
}
],
"views": [],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '4b8a3541fc27086145381f661dbb9a39')"
]
}
}
\ No newline at end of file
...@@ -37,7 +37,7 @@ import com.frostnerd.smokescreen.database.repository.HostSourceRepository ...@@ -37,7 +37,7 @@ import com.frostnerd.smokescreen.database.repository.HostSourceRepository
@Database(entities = [CachedResponse::class, DnsQuery::class, DnsRule::class, HostSource::class], version = AppDatabase.currentVersion) @Database(entities = [CachedResponse::class, DnsQuery::class, DnsRule::class, HostSource::class], version = AppDatabase.currentVersion)
abstract class AppDatabase : RoomDatabase() { abstract class AppDatabase : RoomDatabase() {
companion object { companion object {
const val currentVersion:Int = 11 const val currentVersion:Int = 12
} }
abstract fun cachedResponseDao(): CachedResponseDao abstract fun cachedResponseDao(): CachedResponseDao
......
...@@ -121,11 +121,14 @@ val MIGRATION_10_11 = migration(10, 11) { ...@@ -121,11 +121,14 @@ val MIGRATION_10_11 = migration(10, 11) {
Logger.logIfOpen("DB_MIGRATION", "Migration from 10 to 11 completed") Logger.logIfOpen("DB_MIGRATION", "Migration from 10 to 11 completed")
} }
val MIGRATION_11_12 = migration(11, 12) {
it.execSQL("ALTER TABLE `DnsQuery` ADD COLUMN `isHostBlockedByDnsServer` INTEGER NOT NULL DEFAULT 0")
}
private val INSTANCE by parameterizedLazy<AppDatabase, Context> { private val INSTANCE by parameterizedLazy<AppDatabase, Context> {
Room.databaseBuilder(it, AppDatabase::class.java, "data") Room.databaseBuilder(it, AppDatabase::class.java, "data")
.allowMainThreadQueries() .allowMainThreadQueries()
.addMigrations(MIGRATION_2_X, MIGRATION_3_4, MIGRATION_4_5, MIGRATION_5_6, MIGRATION_6_7, MIGRATION_7_8, MIGRATION_8_9, MIGRATION_9_10, MIGRATION_10_11) .addMigrations(MIGRATION_2_X, MIGRATION_3_4, MIGRATION_4_5, MIGRATION_5_6, MIGRATION_6_7, MIGRATION_7_8, MIGRATION_8_9, MIGRATION_9_10, MIGRATION_10_11, MIGRATION_11_12)
.build() .build()
} }
......
...@@ -25,15 +25,17 @@ import com.google.gson.reflect.TypeToken ...@@ -25,15 +25,17 @@ import com.google.gson.reflect.TypeToken
*/ */
class StringListConverter { class StringListConverter {
private val gson = Gson() private val gson = Gson()
private val type = object:TypeToken<MutableList<String>>() {}.type private val type = object:TypeToken<List<String>>() {}.type
@TypeConverter @TypeConverter
fun stringToList(value: String): MutableList<String> { fun stringToList(value: String): List<String> {
return gson.fromJson(value, type) return if(value == "[]") emptyList()
else gson.fromJson(value, type)
} }
@TypeConverter @TypeConverter
fun someObjectListToString(value: MutableList<String>): String { fun someObjectListToString(value: List<String>): String {
return gson.toJson(value) return if(value.isEmpty()) "[]"
else gson.toJson(value)
} }
} }
\ No newline at end of file
...@@ -36,6 +36,21 @@ interface DnsQueryDao { ...@@ -36,6 +36,21 @@ interface DnsQueryDao {
@Query("SELECT * FROM DnsQuery WHERE name LIKE '%' ||:hostPart || '%'") @Query("SELECT * FROM DnsQuery WHERE name LIKE '%' ||:hostPart || '%'")
fun getAllWithHostLive(hostPart:String): LiveData<List<DnsQuery>> fun getAllWithHostLive(hostPart:String): LiveData<List<DnsQuery>>
@Query("SELECT * FROM DnsQuery WHERE (responseSource='UPSTREAM' AND (:showForwarded=1 OR (:showBlockedByServer=1 AND responseTime>0))) " +
"OR (responseSource='CACHE' AND :showCache=1) " +
"OR (responseSource='LOCALRESOLVER' AND :showDnsRules=1) " +
"OR (responseSource='CACHE_AND_LOCALRESOLVER' AND (:showCache=1 OR :showDnsRules=1)) " +
"OR (responseSource='OTHER' AND :showForwarded=1)")
fun getAllWithFilterLive(showForwarded:Boolean, showCache:Boolean, showDnsRules:Boolean, showBlockedByServer:Boolean):LiveData<List<DnsQuery>>
@Query("SELECT * FROM DnsQuery WHERE (responseSource='UPSTREAM' AND (:showForwarded=1 OR (:showBlockedByServer=1 AND responseTime>0)) " +
"OR (responseSource='CACHE' AND :showCache=1) " +
"OR (responseSource='LOCALRESOLVER' AND :showDnsRules=1) " +
"OR (responseSource='CACHE_AND_LOCALRESOLVER' AND (:showCache=1 OR :showDnsRules=1)) " +
"OR (responseSource='OTHER' AND :showForwarded=1)) " +
"AND name LIKE '%' ||:hostPart || '%'")
fun getAllWithHostAndFilterLive(hostPart:String, showForwarded:Boolean, showCache:Boolean, showDnsRules:Boolean, showBlockedByServer:Boolean):LiveData<List<DnsQuery>>
@Query("SELECT MAX(id) FROM DnsQuery") @Query("SELECT MAX(id) FROM DnsQuery")
fun getLastInsertedId():Long fun getLastInsertedId():Long
......
...@@ -10,6 +10,9 @@ import com.frostnerd.smokescreen.database.converters.DnsTypeConverter ...@@ -10,6 +10,9 @@ import com.frostnerd.smokescreen.database.converters.DnsTypeConverter
import com.frostnerd.smokescreen.database.converters.QuerySourceConverter import com.frostnerd.smokescreen.database.converters.QuerySourceConverter
import com.frostnerd.smokescreen.database.converters.StringListConverter import com.frostnerd.smokescreen.database.converters.StringListConverter
import com.frostnerd.smokescreen.database.recordFromBase64 import com.frostnerd.smokescreen.database.recordFromBase64
import com.frostnerd.smokescreen.equalsAny
import org.minidns.record.A
import org.minidns.record.AAAA
import org.minidns.record.Record import org.minidns.record.Record
/* /*
...@@ -40,13 +43,37 @@ data class DnsQuery( ...@@ -40,13 +43,37 @@ data class DnsQuery(
var responseSource:QueryListener.Source, var responseSource:QueryListener.Source,
val questionTime: Long, val questionTime: Long,
var responseTime: Long = 0, var responseTime: Long = 0,
var responses: MutableList<String> var responses: List<String>,
var isHostBlockedByDnsServer:Boolean = responsesBlockHost(responses)
) { ) {
@delegate:Ignore @delegate:Ignore
val shortName:String by lazy { calculateShortName() } val shortName:String by lazy { calculateShortName() }
@delegate:Ignore
val parsedResponses:List<Record<*>> by lazy {
parseResponses(responses)
}
companion object { companion object {
private val SHORT_DOMAIN_REGEX = "^((?:[^.]{1,3}\\.)+)([\\w]{4,})(?:\\.(?:[^.]*))*\$".toRegex() private val SHORT_DOMAIN_REGEX = "^((?:[^.]{1,3}\\.)+)([\\w]{4,})(?:\\.(?:[^.]*))*\$".toRegex()
private fun responsesBlockHost(responses:List<String>):Boolean {
return parseResponses(responses).any {
(it.type == Record.TYPE.A && (it.payload as A).toString() == "0.0.0.0") ||
(it.type == Record.TYPE.AAAA && (it.payload as AAAA).toString().equalsAny("::0", "0:0:0:0:0:0:0:0"))
}
}
private fun parseResponses(responses:List<String>): List<Record<*>> {
if(responses.isEmpty()) return emptyList()
val parsedResponses = mutableListOf<Record<*>>()
for (response in responses) {
parsedResponses.add(recordFromBase64(response))
}
return parsedResponses
}
fun encodeResponse(record: Record<*>):String = Base64.encodeToString(record.toByteArray(), Base64.NO_WRAP)
} }
/** /**
...@@ -63,16 +90,6 @@ data class DnsQuery( ...@@ -63,16 +90,6 @@ data class DnsQuery(
} else return name } else return name
} }
fun getParsedResponses(): List<Record<*>> {
val responses = mutableListOf<Record<*>>()
for (response in this.responses) {
responses.add(recordFromBase64(response))
}
return responses
}
fun encodeResponse(record: Record<*>):String = Base64.encodeToString(record.toByteArray(), Base64.NO_WRAP)
override fun toString(): String { override fun toString(): String {
return "DnsQuery(id=$id, type=$type, name='$name', askedServer=$askedServer, questionTime=$questionTime, responseTime=$responseTime, responses={${responses.size}})" return "DnsQuery(id=$id, type=$type, name='$name', askedServer=$askedServer, questionTime=$questionTime, responseTime=$responseTime, responses={${responses.size}})"
} }
......
package com.frostnerd.smokescreen.database.repository package com.frostnerd.smokescreen.database.repository
import android.content.Context import android.content.Context
import androidx.lifecycle.LiveData
import androidx.lifecycle.Transformations
import com.frostnerd.smokescreen.database.converters.StringListConverter import com.frostnerd.smokescreen.database.converters.StringListConverter
import com.frostnerd.smokescreen.database.dao.DnsQueryDao import com.frostnerd.smokescreen.database.dao.DnsQueryDao
import com.frostnerd.smokescreen.database.entities.DnsQuery import com.frostnerd.smokescreen.database.entities.DnsQuery
import com.frostnerd.smokescreen.dialog.QueryLogFilterDialog
import com.frostnerd.smokescreen.getPreferences import com.frostnerd.smokescreen.getPreferences
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.GlobalScope
...@@ -34,6 +37,48 @@ import java.io.FileWriter ...@@ -34,6 +37,48 @@ import java.io.FileWriter
class DnsQueryRepository(private val dnsQueryDao: DnsQueryDao) { class DnsQueryRepository(private val dnsQueryDao: DnsQueryDao) {
fun getAllWithFilterLive(filterConfig: QueryLogFilterDialog.FilterConfig): LiveData<List<DnsQuery>> {
return dnsQueryDao.getAllWithFilterLive(filterConfig.showForwarded, filterConfig.showCache, filterConfig.showDnsrules, filterConfig.showBlockedByDns).let {
filterDnsQuery(filterConfig, it)
}
}
fun getAllWithHostAndFilterLive(hostPart:String, filterConfig: QueryLogFilterDialog.FilterConfig): LiveData<List<DnsQuery>> {
return dnsQueryDao.getAllWithHostAndFilterLive(hostPart, filterConfig.showForwarded, filterConfig.showCache, filterConfig.showDnsrules, filterConfig.showBlockedByDns).let {
filterDnsQuery(filterConfig, it)
}
}
private fun filterDnsQuery(filterConfig: QueryLogFilterDialog.FilterConfig, liveData: LiveData<List<DnsQuery>>):LiveData<List<DnsQuery>> {
if(filterConfig.showForwarded && filterConfig.showBlockedByDns) {
return liveData
} else{
val holdsAllData = liveData.value?.let {
dnsQueryDao.getCount() == it.size
} ?: false
return when {
holdsAllData -> {
liveData
}
filterConfig.showForwarded -> {
Transformations.map(liveData) {
it.filter {
!it.isHostBlockedByDnsServer
}
}
}
else -> { // showForwarded = false, showBlockedByDns = true
Transformations.map(liveData) {
it.filter {
it.isHostBlockedByDnsServer
}
}
}
}
}
}
fun insertAllAsync(dnsQueries:List<DnsQuery>): Job { fun insertAllAsync(dnsQueries:List<DnsQuery>): Job {
return GlobalScope.launch(Dispatchers.IO) { return GlobalScope.launch(Dispatchers.IO) {
dnsQueryDao.insertAll(dnsQueries) dnsQueryDao.insertAll(dnsQueries)
......
package com.frostnerd.smokescreen.dialog
import android.content.Context
import androidx.appcompat.app.AlertDialog
import com.frostnerd.smokescreen.R
import com.frostnerd.smokescreen.getPreferences
import kotlinx.android.synthetic.main.dialog_querylog_filter.view.*
/*
* Copyright (C) 2020 Daniel Wolf (Ch4t4r)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* You can contact the developer at daniel.wolf@frostnerd.com.
*/
class QueryLogFilterDialog (
context: Context,
activeFilter:FilterConfig,
shortenDomains:Boolean,
onFilterConfigured:(filterConfig:FilterConfig, shortenDomainsInList:Boolean) -> Unit
): AlertDialog(context, context.getPreferences().theme.dialogStyle) {
init {
val view = layoutInflater.inflate(R.layout.dialog_querylog_filter, null, false)
setTitle(R.string.querylog_filter_title)
setView(view)
setButton(BUTTON_POSITIVE, context.getText(android.R.string.ok)) { _, _ ->
val filter = FilterConfig(
view.showForwarded.isChecked,
view.showCache.isChecked,
view.showDnsRules.isChecked,
view.showBlockedByDns.isChecked
)
onFilterConfigured(filter, view.shortenDomain.isChecked)
}
setButton(BUTTON_NEGATIVE, context.getString(android.R.string.cancel)) { _, _ -> }
view.showForwarded.isChecked = activeFilter.showForwarded
view.showCache.isChecked = activeFilter.showCache
view.showDnsRules.isChecked = activeFilter.showDnsrules
view.showBlockedByDns.isChecked = activeFilter.showBlockedByDns